Merge lp:~corey.bryant/charm-helpers/systemd-fixup-and-upper-constraints into lp:charm-helpers

[Corey Bryant]

This mp has a number of updates for openstack deploy from source that are explained in the commit messages.

[lp:~corey.bryant/charm-helpers/systemd-fixup-and-upper-constraints]

599. By Corey Bryant on 2016-06-30

Merge lp:charm-helpers

=== modified file 'charmhelpers/contrib/openstack/utils.py'
--- charmhelpers/contrib/openstack/utils.py	2016-06-15 20:59:51 +0000
+++ charmhelpers/contrib/openstack/utils.py	2016-06-30 12:59:15 +0000
@@ -222,7 +222,6 @@
 }
 
 GIT_DEFAULT_BRANCHES = {
-    'icehouse': 'icehouse-eol',
     'kilo': 'stable/kilo',
     'liberty': 'stable/liberty',
     'mitaka': 'stable/mitaka',
@@ -743,12 +742,18 @@
             }
             repos = [repo]
 
+            # NOTE(coreycb): This is a temp work-around until the requirements
+            # repo moves from stable/kilo branch to kilo-eol tag. The core
+            # repos have already done this.
+            if default == 'kilo':
+                branch = 'kilo-eol'
+
             # neutron-* and nova-* charms require some additional repos
             if service in ['neutron-api', 'neutron-gateway',
                            'neutron-openvswitch']:
                 core_project = 'neutron'
                 for project in ['neutron-fwaas', 'neutron-lbaas',
-                                'neutron-vpnaas']:
+                                'neutron-vpnaas', 'nova']:
                     repo = {
                         'name': project,
                         'repository': GIT_DEFAULT_REPOS[project],
@@ -837,6 +842,7 @@
         pip_install(p, upgrade=True, proxy=http_proxy,
                     venv=os.path.join(parent_dir, 'venv'))
 
+    constraints = None
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
@@ -848,10 +854,15 @@
                                                      parent_dir, http_proxy,
                                                      update_requirements=False)
             requirements_dir = repo_dir
+            constraints = os.path.join(repo_dir, "upper-constraints.txt")
+            # upper-constraints didn't exist until after icehouse
+            if not os.path.isfile(constraints):
+                constraints = None
         else:
             repo_dir = _git_clone_and_install_single(repo, branch, depth,
                                                      parent_dir, http_proxy,
-                                                     update_requirements=True)
+                                                     update_requirements=True,
+                                                     constraints=constraints)
 
     os.environ = old_environ
 
@@ -883,7 +894,7 @@
 
 
 def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
-                                  update_requirements):
+                                  update_requirements, constraints=None):
     """
     Clone and install a single git repository.
     """
@@ -906,9 +917,10 @@
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
     if http_proxy:
-        pip_install(repo_dir, proxy=http_proxy, venv=venv)
+        pip_install(repo_dir, proxy=http_proxy, venv=venv,
+                    constraints=constraints)
     else:
-        pip_install(repo_dir, venv=venv)
+        pip_install(repo_dir, venv=venv, constraints=constraints)
 
     return repo_dir
 
@@ -988,6 +1000,7 @@
     script generation, which is used by the OpenStack packages.
     """
     for f in os.listdir(templates_dir):
+        # Create the init script and systemd unit file from the template
         if f.endswith(".init.in"):
             init_in_file = f
             init_file = f[:-8]
@@ -1013,10 +1026,47 @@
                 os.remove(init_dest)
             if os.path.exists(service_dest):
                 os.remove(service_dest)
-            shutil.move(init_source, init_dest)
-            shutil.move(service_source, service_dest)
+            shutil.copyfile(init_source, init_dest)
+            shutil.copyfile(service_source, service_dest)
             os.chmod(init_dest, 0o755)
 
+    for f in os.listdir(templates_dir):
+        # If there's a service.in file, use it instead of the generated one
+        if f.endswith(".service.in"):
+            service_in_file = f
+            service_file = f[:-3]
+
+            service_in_source = os.path.join(templates_dir, service_in_file)
+            service_source = os.path.join(templates_dir, service_file)
+            service_dest = os.path.join('/lib/systemd/system', service_file)
+
+            shutil.copyfile(service_in_source, service_source)
+
+            if os.path.exists(service_dest):
+                os.remove(service_dest)
+            shutil.copyfile(service_source, service_dest)
+
+    for f in os.listdir(templates_dir):
+        # Generate the systemd unit if there's no existing .service.in
+        if f.endswith(".init.in"):
+            init_in_file = f
+            init_file = f[:-8]
+            service_in_file = "{}.service.in".format(init_file)
+            service_file = "{}.service".format(init_file)
+
+            init_in_source = os.path.join(templates_dir, init_in_file)
+            service_in_source = os.path.join(templates_dir, service_in_file)
+            service_source = os.path.join(templates_dir, service_file)
+            service_dest = os.path.join('/lib/systemd/system', service_file)
+
+            if not os.path.exists(service_in_source):
+                cmd = ['pkgos-gen-systemd-unit', init_in_source]
+                subprocess.check_call(cmd)
+
+                if os.path.exists(service_dest):
+                    os.remove(service_dest)
+                shutil.copyfile(service_source, service_dest)
+
 
 def os_workload_status(configs, required_interfaces, charm_func=None):
     """

=== modified file 'charmhelpers/contrib/python/packages.py'
--- charmhelpers/contrib/python/packages.py	2016-02-02 10:46:04 +0000
+++ charmhelpers/contrib/python/packages.py	2016-06-30 12:59:15 +0000
@@ -80,7 +80,8 @@
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
+def pip_install(package, fatal=False, upgrade=False, venv=None,
+                constraints=None, **options):
     """Install a python package"""
     if venv:
         venv_python = os.path.join(venv, 'bin/pip')
@@ -95,6 +96,9 @@
     if upgrade:
         command.append('--upgrade')
 
+    if constraints:
+        command.extend(['-c', constraints])
+
     if isinstance(package, list):
         command.extend(package)
     else:

=== modified file 'tests/contrib/openstack/test_openstack_utils.py'
--- tests/contrib/openstack/test_openstack_utils.py	2016-05-27 09:05:48 +0000
+++ tests/contrib/openstack/test_openstack_utils.py	2016-06-30 12:59:15 +0000
@@ -730,13 +730,16 @@
         error_out.assert_called_with(
             'openstack-origin-git key \'%s\' is missing' % key)
 
+    @patch('os.path.isfile')
     @patch('os.path.join')
     @patch.object(openstack, 'error_out')
     @patch.object(openstack, '_git_clone_and_install_single')
     @patch.object(openstack, 'pip_install')
     @patch.object(openstack, 'pip_create_virtualenv')
     def test_git_clone_and_install_errors(self, pip_venv, pip_install,
-                                          git_install_single, error_out, join):
+                                          git_install_single, error_out, join,
+                                          isfile):
+        isfile.return_value = True
         git_missing_repos = """
           repostories:
              - {name: requirements,
@@ -809,13 +812,13 @@
         charm_dir.return_value = '/var/lib/juju/units/testing-foo-0/charm'
         # the following sets the global requirements_dir
         _git_install_single.return_value = '/mnt/openstack-git/requirements'
-        join.return_value = '/mnt/openstack-git/venv'
+        join.return_value = 'joined-path'
 
         openstack.git_clone_and_install(openstack_origin_git, proj)
         self.assertTrue(pip_venv.called)
         pip_install.assert_called_with('setuptools', upgrade=True,
                                        proxy=None,
-                                       venv='/mnt/openstack-git/venv')
+                                       venv='joined-path')
         self.assertTrue(_git_install_single.call_count == 2)
         expected = [
             call('git://git.openstack.org/openstack/requirements',
@@ -823,7 +826,7 @@
                  update_requirements=False),
             call('git://git.openstack.org/openstack/keystone',
                  'stable/juno', '1', '/mnt/openstack-git', None,
-                 update_requirements=True)
+                 update_requirements=True, constraints=None)
         ]
         self.assertEquals(expected, _git_install_single.call_args_list)
         assert not error_out.called
@@ -855,7 +858,8 @@
                                           branch=branch)
         assert not _git_update_reqs.called
         pip_install.assert_called_with(dest_dir, venv='/mnt/openstack-git',
-                                       proxy='http://squid-proxy-url')
+                                       proxy='http://squid-proxy-url',
+                                       constraints=None)
 
     @patch('os.path.join')
     @patch('os.mkdir')
@@ -887,7 +891,8 @@
                                           branch=branch)
         _git_update_reqs.assert_called_with(venv_dir, dest_dir, reqs_dir)
         pip_install.assert_called_with(dest_dir, venv='/mnt/openstack-git',
-                                       proxy='http://squid-proxy-url')
+                                       proxy='http://squid-proxy-url',
+                                       constraints=None)
 
     @patch('os.path.join')
     @patch('os.getcwd')