Merge ~afreiberger/charm-juju-lxd:fix_blocked_state into charm-juju-lxd:master

[Drew Freiberger]

f6bb5ed... by Drew Freiberger on 2020-04-29

Added sysctl and limits.conf settings & git proxy

Git repos weren't cloning when using juju-http-proxy in the model
configs, so I've added in somethign that will set the global git proxy
for root before trying to clone openstack-on-lxd.

I've also added in sysctl and security limits configs recommended for
running as many lxd containers and services as we will be for juju on
lxd (especially for openstack-on-lxd).

9ee569e... by Drew Freiberger on 2020-05-01

Added pam_limits.so to common-session* files

The /etc/security/limits.d/ files were not being read by all sessions
(sudo/systemd) which was continuing to cause resource starvation with HA
openstack-on-lxd clouds. Added lines to append the session limits to
common-session and common-session-noninteractive files.

Static variable cleanup, moved to top of reactive hooks file.

[Zachary Zehring (zzehring)]

LGTM +1.

review: Approve

d0762e3... by Drew Freiberger on 2020-05-01

Update verbiage and exception handling for lxd-group missing

Unmerged commits

d0762e3... by Drew Freiberger on 2020-05-01

Update verbiage and exception handling for lxd-group missing

9ee569e... by Drew Freiberger on 2020-05-01

Added pam_limits.so to common-session* files

The /etc/security/limits.d/ files were not being read by all sessions
(sudo/systemd) which was continuing to cause resource starvation with HA
openstack-on-lxd clouds. Added lines to append the session limits to
common-session and common-session-noninteractive files.

Static variable cleanup, moved to top of reactive hooks file.

f6bb5ed... by Drew Freiberger on 2020-04-29

Added sysctl and limits.conf settings & git proxy

Git repos weren't cloning when using juju-http-proxy in the model
configs, so I've added in somethign that will set the global git proxy
for root before trying to clone openstack-on-lxd.

I've also added in sysctl and security limits configs recommended for
running as many lxd containers and services as we will be for juju on
lxd (especially for openstack-on-lxd).

d195e34... by Drew Freiberger on 2020-04-29

Updated reactive framework to re-try missing group

There was an issue with the juju_lxd_configure function that was running
finally step even when groups were missing, setting the
juju-lxd.configured flag and skipping update-status checks for group
being present. This patch fixes this race condition.

diff --git a/files/50-security-limits.conf b/files/50-security-limits.conf
new file mode 100644
index 0000000..bee1301
--- /dev/null
+++ b/files/50-security-limits.conf
@@ -0,0 +1,6 @@
+* soft nofile 1048576
+* hard nofile 1048576
+root soft nofile 1048576
+root hard nofile 1048576
+* soft memlock unlimited
+* hard memlock unlimited
diff --git a/files/50-sysctl.conf b/files/50-sysctl.conf
new file mode 100644
index 0000000..db33cfe
--- /dev/null
+++ b/files/50-sysctl.conf
@@ -0,0 +1,8 @@
+fs.inotify.max_queued_events=1048576
+fs.inotify.max_user_instances = 1048576
+fs.inotify.max_user_watches = 1048576
+vm.max_map_count=262144
+kernel.dmesg_restrict=1
+net.ipv4.neigh.default.gc_thresh3=8192
+net.ipv6.neigh.default.gc_thresh3=8192
+kernel.keys.maxkeys=2000
diff --git a/reactive/juju_lxd.py b/reactive/juju_lxd.py
index 1c05993..e980f21 100644
--- a/reactive/juju_lxd.py
+++ b/reactive/juju_lxd.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 # This file is part of juju_lxd charm which ensures that snap and the required
 # Ubuntu packages for our charmlab are installed.
@@ -16,24 +16,45 @@
 # You should have received a copy of the GNU General Public License along with
 # this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from charms.reactive import when, when_not, set_state, remove_state
+from charms.reactive import when, when_any, when_not, set_state, remove_state
 from charms.reactive.helpers import data_changed
 from charmhelpers.core import hookenv
 from charmhelpers.core.host import group_exists
+from shutil import copyfile
 import subprocess
 import os
 import grp
 
 
+LIMITS_FILE = '/etc/security/limits.d/50-juju-lxd.conf'
+SYSCTL_FILE = '/etc/sysctl.d/50-juju-lxd.conf'
+PAM_SESSION_LIMITS = 'session  required pam_limits.so'
+PAM_SESSION_FILE = '/etc/pam.d/common-session'
+PAM_SESSION_NI_FILE = '/etc/pam.d/common-session-noninteractive'
+
+
 @when_not('juju-lxd.installed')
 @when('apt.installed.zfsutils-linux', 'apt.installed.git')
 def install_charm_juju_lxd():
     git_repo()
+    setup_sysctl()
+    setup_limits()
     lxd_init()
     status_set("active", "Ready")
     set_state('juju-lxd.installed')
 
 
+def setup_sysctl():
+    copyfile('files/50-sysctl.conf', SYSCTL_FILE)
+    subprocess.call(['sysctl', '-p'])
+
+
+def setup_limits():
+    copyfile('files/50-limits.conf', LIMITS_FILE)
+    subprocess.call('echo {} >> {}'.format(PAM_SESSION_LIMITS, PAM_SESSION_FILE), shell=True)
+    subprocess.call('echo {} >> {}'.format(PAM_SESSION_LIMITS, PAM_SESSION_NI_FILE), shell=True)
+
+
 def status_set(state, message):
     '''Set the unit's workload status.'''
     (current_state, current_message) = hookenv.status_get()
@@ -46,7 +67,7 @@ def status_set(state, message):
     hookenv.log('{}: {}'.format(state, message), lvl)
 
 
-@when('juju-lxd.installed')
+@when_any('juju-lxd.installed', 'juju-lxd.missing-group')
 @when_not('juju-lxd.configured')
 def juju_lxd_configure():
     config = hookenv.config()
@@ -58,13 +79,16 @@ def juju_lxd_configure():
             sync_groups(src_groups, 'lxd')
         except KeyError:
             status_set('blocked',
-                       'lxd-group "{0}" does not exist'.format(src_groups))
-        finally:
+                       'one or more lxd-source groups "{0}" do not exist'.format(config.get('lxd-group')))
+            set_state('juju-lxd.missing-group')
+        else:
             data_changed('juju-lxd.config', config)
             set_state('juju-lxd.configured')
+            remove_state('juju-lxd.missing-group')
             status_set('active', 'Ready')
     else:
         status_set('blocked', 'LXD group "{0}" does not exist'.format(dst_group))
+        set_state('juju-lxd.missing-group')
 
 
 @when('juju-lxd.configured')
@@ -91,7 +115,6 @@ def lxd_init():
     else:
         subprocess.call(['lxd', 'init', '--auto', '--storage-backend', 'dir'])
 
-
     subprocess.call(['lxc', 'network', 'delete', 'lxdbr0'])
     subprocess.call(['lxc', 'network', 'create', 'lxdbr0',
                      'ipv4.address=10.0.8.1/24', 'ipv6.address=none',
@@ -103,7 +126,7 @@ def lxd_init():
 
     try:
         subprocess.call(['lxc', 'profile', 'create', 'juju-default'])
-    except:
+    except subprocess.CalledProcessError:
         pass
 
     if use_zfs:
@@ -114,17 +137,20 @@ def lxd_init():
         try:
             subprocess.check_output(['lxc', 'profile', 'edit', 'juju-default'],
                                     stdin=profileyaml)
-        except:
+        except subprocess.CalledProcessError:
             pass
 
 
 def git_repo():
+    if os.environ['JUJU_CHARM_HTTP_PROXY']:
+        subprocess.call(['git', 'config', '--global', os.environ['JUJU_CHARM_HTTP_PROXY']])
+
     subprocess.call(['git', 'clone',
                      'https://github.com/openstack-charmers/openstack-on-lxd'])
     try:
         os.symlink("{}/openstack-on-lxd".format(hookenv.charm_dir()),
                    '/home/ubuntu/openstack-on-lxd')
-    except:
+    except OSError:
         pass
 
 
@@ -140,4 +166,3 @@ def sync_groups(src_groups, dst):
     users = ','.join(list(set(members)))
     status_set("maintenance", "configuring users in lxd group")
     subprocess.call(['gpasswd', '-M', users, dst])
-