Merge lp:~corey.bryant/charms/trusty/keystone/git-ods into lp:~openstack-charmers-archive/charms/trusty/keystone/next

[Corey Bryant]

Adds support to:
* Use a Python virtualenv when deploying from source
* Default to shallow clone for git (depth=1)
* Pass http_proxy to pip installs if specified

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #3821 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/3821/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #3608 keystone-next for corey.bryant mp257188
    UNIT FAIL: unit-test failed

UNIT Results (max last 2 lines):
make: *** [unit_test] Error 1
ERROR:root:Make target returned non-zero.

Full unit test output: http://paste.ubuntu.com/10868274/
Build: http://10.245.162.77:8080/job/charm_unit_test/3608/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #3607 keystone-next for corey.bryant mp257188
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/3607/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #3830 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/3830/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #3617 keystone-next for corey.bryant mp257188
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/3617/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #3616 keystone-next for corey.bryant mp257188
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/3616/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #3865 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/3865/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #3652 keystone-next for corey.bryant mp257188
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/3652/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #3874 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/3874/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #3661 keystone-next for corey.bryant mp257188
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/3661/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #3656 keystone-next for corey.bryant mp257188
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/3656/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #3905 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/3905/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #3692 keystone-next for corey.bryant mp257188
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/3692/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #4258 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/4258/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #3984 keystone-next for corey.bryant mp257188
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/3984/

[lp:~corey.bryant/charms/trusty/keystone/git-ods]

181. By Corey Bryant on 2015-05-11

Sync charm-helpers

182. By Corey Bryant on 2015-05-11

Add comment to fix bin symlinks

183. By Corey Bryant on 2015-05-11

Sync charm-helpers

184. By Corey Bryant on 2015-05-12

Sync charm-helpers

185. By Corey Bryant on 2015-05-12

Clone from github in deploy from source amulet tests

186. By Corey Bryant on 2015-05-12

Add libyaml-dev as base git package

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #4464 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/4464/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #4189 keystone-next for corey.bryant mp257188
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/4189/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4087 keystone-next for corey.bryant mp257188
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/4087/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #4526 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/4526/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #4251 keystone-next for corey.bryant mp257188
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/4251/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4105 keystone-next for corey.bryant mp257188
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/4105/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #4534 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/4534/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #4259 keystone-next for corey.bryant mp257188
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/4259/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4113 keystone-next for corey.bryant mp257188
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/4113/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #4543 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/4543/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #4268 keystone-next for corey.bryant mp257188
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/4268/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4122 keystone-next for corey.bryant mp257188
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/4122/

[lp:~corey.bryant/charms/trusty/keystone/git-ods]

187. By Corey Bryant on 2015-05-27

Sync charm-helpers

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #4838 keystone-next for corey.bryant mp257188
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/4838/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #4518 keystone-next for corey.bryant mp257188
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/4518/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4329 keystone-next for corey.bryant mp257188
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/4329/

=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2015-04-16 14:09:47 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2015-05-27 13:06:53 +0000
@@ -53,9 +53,13 @@
     get_ipv6_addr
 )
 
+from charmhelpers.contrib.python.packages import (
+    pip_create_virtualenv,
+    pip_install,
+)
+
 from charmhelpers.core.host import lsb_release, mounts, umount
 from charmhelpers.fetch import apt_install, apt_cache, install_remote
-from charmhelpers.contrib.python.packages import pip_install
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 
@@ -497,7 +501,17 @@
 requirements_dir = None
 
 
-def git_clone_and_install(projects_yaml, core_project):
+def _git_yaml_load(projects_yaml):
+    """
+    Load the specified yaml into a dictionary.
+    """
+    if not projects_yaml:
+        return None
+
+    return yaml.load(projects_yaml)
+
+
+def git_clone_and_install(projects_yaml, core_project, depth=1):
     """
     Clone/install all specified OpenStack repositories.
 
@@ -510,23 +524,22 @@
              repository: 'git://git.openstack.org/openstack/requirements.git',
              branch: 'stable/icehouse'}
         directory: /mnt/openstack-git
-        http_proxy: http://squid.internal:3128
-        https_proxy: https://squid.internal:3128
+        http_proxy: squid-proxy-url
+        https_proxy: squid-proxy-url
 
         The directory, http_proxy, and https_proxy keys are optional.
     """
     global requirements_dir
     parent_dir = '/mnt/openstack-git'
-
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    http_proxy = None
+
+    projects = _git_yaml_load(projects_yaml)
     _git_validate_projects_yaml(projects, core_project)
 
     old_environ = dict(os.environ)
 
     if 'http_proxy' in projects.keys():
+        http_proxy = projects['http_proxy']
         os.environ['http_proxy'] = projects['http_proxy']
     if 'https_proxy' in projects.keys():
         os.environ['https_proxy'] = projects['https_proxy']
@@ -534,15 +547,19 @@
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
 
+    pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
+
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
         if p['name'] == 'requirements':
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=False)
             requirements_dir = repo_dir
         else:
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=True)
 
     os.environ = old_environ
@@ -574,7 +591,8 @@
         error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
 
-def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
+def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
+                                  update_requirements):
     """
     Clone and install a single git repository.
     """
@@ -587,7 +605,8 @@
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
+                                  depth=depth)
     else:
         repo_dir = dest_dir
 
@@ -598,7 +617,12 @@
         _git_update_requirements(repo_dir, requirements_dir)
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
-    pip_install(repo_dir)
+    if http_proxy:
+        pip_install(repo_dir, proxy=http_proxy,
+                    venv=os.path.join(parent_dir, 'venv'))
+    else:
+        pip_install(repo_dir,
+                    venv=os.path.join(parent_dir, 'venv'))
 
     return repo_dir
 
@@ -621,16 +645,27 @@
     os.chdir(orig_dir)
 
 
+def git_pip_venv_dir(projects_yaml):
+    """
+    Return the pip virtualenv path.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    projects = _git_yaml_load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    return os.path.join(parent_dir, 'venv')
+
+
 def git_src_dir(projects_yaml, project):
     """
     Return the directory where the specified project's source is located.
     """
     parent_dir = '/mnt/openstack-git'
 
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    projects = _git_yaml_load(projects_yaml)
 
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
@@ -640,3 +675,15 @@
             return os.path.join(parent_dir, os.path.basename(p['repository']))
 
     return None
+
+
+def git_yaml_value(projects_yaml, key):
+    """
+    Return the value in projects_yaml for the specified key.
+    """
+    projects = _git_yaml_load(projects_yaml)
+
+    if key in projects.keys():
+        return projects[key]
+
+    return None

=== modified file 'hooks/charmhelpers/contrib/python/packages.py'
--- hooks/charmhelpers/contrib/python/packages.py	2015-03-11 11:45:09 +0000
+++ hooks/charmhelpers/contrib/python/packages.py	2015-05-27 13:06:53 +0000
@@ -17,8 +17,11 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+import subprocess
+
 from charmhelpers.fetch import apt_install, apt_update
-from charmhelpers.core.hookenv import log
+from charmhelpers.core.hookenv import charm_dir, log
 
 try:
     from pip import main as pip_execute
@@ -51,11 +54,15 @@
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, **options):
+def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
     """Install a python package"""
-    command = ["install"]
+    if venv:
+        venv_python = os.path.join(venv, 'bin/pip')
+        command = [venv_python, "install"]
+    else:
+        command = ["install"]
 
-    available_options = ('proxy', 'src', 'log', "index-url", )
+    available_options = ('proxy', 'src', 'log', 'index-url', )
     for option in parse_options(options, available_options):
         command.append(option)
 
@@ -69,7 +76,10 @@
 
     log("Installing {} package with options: {}".format(package,
                                                         command))
-    pip_execute(command)
+    if venv:
+        subprocess.check_call(command)
+    else:
+        pip_execute(command)
 
 
 def pip_uninstall(package, **options):
@@ -94,3 +104,16 @@
     """Returns the list of current python installed packages
     """
     return pip_execute(["list"])
+
+
+def pip_create_virtualenv(path=None):
+    """Create an isolated Python environment."""
+    apt_install('python-virtualenv')
+
+    if path:
+        venv_path = path
+    else:
+        venv_path = os.path.join(charm_dir(), 'venv')
+
+    if not os.path.exists(venv_path):
+        subprocess.check_call(['virtualenv', venv_path])

=== modified file 'hooks/charmhelpers/fetch/giturl.py'
--- hooks/charmhelpers/fetch/giturl.py	2015-03-11 11:45:09 +0000
+++ hooks/charmhelpers/fetch/giturl.py	2015-05-27 13:06:53 +0000
@@ -45,14 +45,16 @@
         else:
             return True
 
-    def clone(self, source, dest, branch):
+    def clone(self, source, dest, branch, depth=None):
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
 
-        repo = Repo.clone_from(source, dest)
-        repo.git.checkout(branch)
+        if depth:
+            Repo.clone_from(source, dest, branch=branch, depth=depth)
+        else:
+            Repo.clone_from(source, dest, branch=branch)
 
-    def install(self, source, branch="master", dest=None):
+    def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
         if dest:
@@ -63,7 +65,7 @@
         if not os.path.exists(dest_dir):
             mkdir(dest_dir, perms=0o755)
         try:
-            self.clone(source, dest_dir, branch)
+            self.clone(source, dest_dir, branch, depth)
         except GitCommandError as e:
             raise UnhandledSource(e.message)
         except OSError as e:

=== modified file 'hooks/keystone_utils.py'
--- hooks/keystone_utils.py	2015-05-08 11:43:00 +0000
+++ hooks/keystone_utils.py	2015-05-27 13:06:53 +0000
@@ -44,8 +44,15 @@
     git_install_requested,
     git_clone_and_install,
     git_src_dir,
+    git_yaml_value,
+    git_pip_venv_dir,
     os_release,
-    save_script_rc as _save_script_rc)
+    save_script_rc as _save_script_rc
+)
+
+from charmhelpers.contrib.python.packages import (
+    pip_install,
+)
 
 from charmhelpers.core.strutils import (
     bool_from_string,
@@ -123,9 +130,11 @@
 
 BASE_GIT_PACKAGES = [
     'libffi-dev',
+    'libmysqlclient-dev',
     'libssl-dev',
     'libxml2-dev',
     'libxslt1-dev',
+    'libyaml-dev',
     'python-dev',
     'python-pip',
     'python-setuptools',
@@ -139,7 +148,6 @@
 # ubuntu packages that should not be installed when deploying from git
 GIT_PACKAGE_BLACKLIST = [
     'keystone',
-    'python-keystoneclient',
 ]
 
 API_PORTS = {
@@ -1700,6 +1708,14 @@
 
 def git_post_install(projects_yaml):
     """Perform keystone post-install setup."""
+    http_proxy = git_yaml_value(projects_yaml, 'http_proxy')
+    if http_proxy:
+        pip_install('mysql-python', proxy=http_proxy,
+                    venv=git_pip_venv_dir(projects_yaml))
+    else:
+        pip_install('mysql-python',
+                    venv=git_pip_venv_dir(projects_yaml))
+
     src_etc = os.path.join(git_src_dir(projects_yaml, 'keystone'), 'etc')
     configs = {
         'src': src_etc,
@@ -1710,15 +1726,28 @@
         shutil.rmtree(configs['dest'])
     shutil.copytree(configs['src'], configs['dest'])
 
+    # NOTE(coreycb): Need to find better solution than bin symlinks.
+    symlinks = [
+        {'src': os.path.join(git_pip_venv_dir(projects_yaml),
+                             'bin/keystone-manage'),
+         'link': '/usr/local/bin/keystone-manage'},
+    ]
+
+    for s in symlinks:
+        if os.path.lexists(s['link']):
+            os.remove(s['link'])
+        os.symlink(s['src'], s['link'])
+
     render('git/logging.conf', '/etc/keystone/logging.conf', {}, perms=0o644)
 
+    bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
     keystone_context = {
         'service_description': 'Keystone API server',
         'service_name': 'Keystone',
         'user_name': 'keystone',
         'start_dir': '/var/lib/keystone',
         'process_name': 'keystone',
-        'executable_name': '/usr/local/bin/keystone-all',
+        'executable_name': os.path.join(bin_dir, 'keystone-all'),
         'config_files': ['/etc/keystone/keystone.conf'],
         'log_file': '/var/log/keystone/keystone.log',
     }

=== modified file 'tests/basic_deployment.py'
--- tests/basic_deployment.py	2015-05-13 09:42:17 +0000
+++ tests/basic_deployment.py	2015-05-27 13:06:53 +0000
@@ -59,10 +59,10 @@
             openstack_origin_git = {
                 'repositories': [
                     {'name': 'requirements',
-                     'repository': 'git://git.openstack.org/openstack/requirements',
+                     'repository': 'git://github.com/openstack/requirements',
                      'branch': branch},
                     {'name': 'keystone',
-                     'repository': 'git://git.openstack.org/openstack/keystone',
+                     'repository': 'git://github.com/openstack/keystone',
                      'branch': branch},
                 ],
                 'directory': '/mnt/openstack-git',

=== modified file 'unit_tests/test_keystone_hooks.py'
--- unit_tests/test_keystone_hooks.py	2015-04-17 12:10:54 +0000
+++ unit_tests/test_keystone_hooks.py	2015-05-27 13:06:53 +0000
@@ -117,10 +117,11 @@
         self.configure_installation_source.assert_called_with(repo)
         self.assertTrue(self.apt_update.called)
         self.apt_install.assert_called_with(
-            ['haproxy', 'unison', 'python-setuptools', 'python-six', 'uuid',
-             'python-mysqldb', 'python-pip', 'libssl-dev', 'openssl',
-             'libffi-dev', 'apache2', 'pwgen', 'libxslt1-dev',
-             'python-psycopg2', 'zlib1g-dev', 'python-dev', 'libxml2-dev'],
+            ['haproxy', 'unison', 'python-setuptools', 'python-keystoneclient',
+             'uuid', 'python-mysqldb', 'libmysqlclient-dev', 'libssl-dev',
+             'openssl', 'libffi-dev', 'apache2', 'python-pip', 'pwgen',
+             'python-six', 'libxslt1-dev', 'python-psycopg2', 'libyaml-dev',
+             'zlib1g-dev', 'python-dev', 'libxml2-dev'],
             fatal=True)
         self.git_install.assert_called_with(projects_yaml)
 

=== modified file 'unit_tests/test_keystone_utils.py'
--- unit_tests/test_keystone_utils.py	2015-04-17 12:10:54 +0000
+++ unit_tests/test_keystone_utils.py	2015-05-27 13:06:53 +0000
@@ -42,6 +42,7 @@
     'https',
     'is_relation_made',
     'peer_store',
+    'pip_install',
     # generic
     'apt_update',
     'apt_upgrade',
@@ -656,26 +657,35 @@
     @patch.object(utils, 'git_src_dir')
     @patch.object(utils, 'service_restart')
     @patch.object(utils, 'render')
+    @patch.object(utils, 'git_pip_venv_dir')
     @patch('os.path.join')
     @patch('os.path.exists')
+    @patch('os.symlink')
     @patch('shutil.copytree')
     @patch('shutil.rmtree')
-    def test_git_post_install(self, rmtree, copytree, exists, join, render,
-                              service_restart, git_src_dir):
+    @patch('subprocess.check_call')
+    def test_git_post_install(self, check_call, rmtree, copytree, symlink,
+                              exists, join, venv, render, service_restart,
+                              git_src_dir):
         projects_yaml = openstack_origin_git
         join.return_value = 'joined-string'
+        venv.return_value = '/mnt/openstack-git/venv'
         utils.git_post_install(projects_yaml)
         expected = [
             call('joined-string', '/etc/keystone'),
         ]
         copytree.assert_has_calls(expected)
+        expected = [
+            call('joined-string', '/usr/local/bin/keystone-manage'),
+        ]
+        symlink.assert_has_calls(expected, any_order=True)
         keystone_context = {
             'service_description': 'Keystone API server',
             'service_name': 'Keystone',
             'user_name': 'keystone',
             'start_dir': '/var/lib/keystone',
             'process_name': 'keystone',
-            'executable_name': '/usr/local/bin/keystone-all',
+            'executable_name': 'joined-string',
             'config_files': ['/etc/keystone/keystone.conf'],
             'log_file': '/var/log/keystone/keystone.log',
         }