Merge bootstack-ops:mtu-scripts into bootstack-ops:master

[James Hebden]

Add scripts for fixing/changing and verifying MTU on BootStack clouds

[Peter Sabaini (peter-sabaini)]

Thanks, see some inline comments

[James Hebden (ec0)]

Thanks for checking this out Peter - I've added capture of return code and stdout and stderr for failed pings, and print a summary at the end of the check-mtus script, for better feedback when diagnosing MTU issues.

I stuck with standard ping though, as I'd like to avoid needing to install fping to run this script. I also found some inconsistencies in it's output when pinging hosts with incorrect MTUs that made it a lot harder to get good status information out of it. It was dumping a failure for every host, whilst simultaneously dumping success messages to stdout for the OK hosts.

[Peter Sabaini (peter-sabaini)]

Ack, thanks. I'm still a fping fan but if it works with ping I'm all +1 :-)

review: Approve

diff --git a/ops-bundle/scripts/check-mtus.py b/ops-bundle/scripts/check-mtus.py
new file mode 100755
index 0000000..9c4710f
--- /dev/null
+++ b/ops-bundle/scripts/check-mtus.py
@@ -0,0 +1,94 @@
+#!/usr/bin/python
+import sys
+import yaml
+import subprocess
+
+mtu = sys.argv[1] if len(sys.argv) > 1 else 8900
+print("Checking hosts reachable at MTU of %s" % mtu)
+
+global ping_errors, statistics
+
+ping_size = int(mtu) - 28
+statistics = {'hosts': 0, 
+              'containers': 0, 
+              'ok': 0.0, 
+              'error': 0.0,
+              'total': 0.0}
+ping_errors = []
+
+juju_status = yaml.load(subprocess.check_output(['juju', 'status', '--format=yaml']))
+
+
+def summary():
+
+    global statistics, ping_errors
+
+    print('Test Summary:')
+    print("  Number of metals: %s" % statistics['hosts'])
+    print("  Number of containers: %s" % statistics['containers'])
+    print('  Ping success: %d%% (\033[92m%d\033[0m,\033[91m%d\033[0m)' %
+          ((statistics['ok'] / statistics['total']) * 100, 
+           statistics['ok'], statistics['error']))
+
+    for ping_error in ping_errors:
+        print('')
+        print('\033[91mPing failure:\033[0m %s (%s) rc %d' % (ping_error['host'], 
+                                                              ping_error['ip'],
+                                                              ping_error['rc']))
+        print(ping_error['stderr'])
+        print(ping_error['stdout'])
+
+
+def green_pass():
+    print('\033[92m' + "OK" + '\033[0m')
+
+
+def red_fail():
+    print('\033[91m' + "FAIL" + '\033[0m')  
+
+
+def ping(ip):
+    ping_command = 'ping'
+    if ':' in ip:
+        ping_command = 'ping6'
+    ping_proc = subprocess.Popen([ping_command, "-c", "1", "-W", "1", "-s", str(ping_size), ip],
+                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                 close_fds=True)
+    out, err = ping_proc.communicate()
+    return [ping_proc.returncode, out, err]
+
+
+def ping_result(host, ip):
+
+    global statistics, ping_errors 
+
+    print("Checking %s (%s)" % (host, ip)),
+    ping_res = ping(ip)
+    statistics['total'] += 1
+
+    if ping_res[0] == 0:
+        statistics['ok'] += 1
+        green_pass()
+    else:
+        statistics['error'] += 1
+        ping_errors.append({'host': host, 'ip': ip, 
+                            'rc': ping_res[0],
+                            'stdout': ping_res[1],
+                            'stderr': ping_res[2]}) 
+        red_fail()
+
+
+for key, machine in juju_status['machines'].iteritems():
+
+    host = machine['instance-id']
+    statistics['hosts'] += 1
+    for ip in machine['ip-addresses']:
+        ping_result(host, ip)
+    if 'containers' in machine:
+        for key, container in machine['containers'].iteritems():
+            host = container['instance-id']
+            statistics['containers'] += 1
+            for ip in container['ip-addresses']:
+                ping_result(host, ip)
+
+summary()
diff --git a/ops-bundle/scripts/fix-mtus.py b/ops-bundle/scripts/fix-mtus.py
new file mode 100755
index 0000000..693eca2
--- /dev/null
+++ b/ops-bundle/scripts/fix-mtus.py
@@ -0,0 +1,77 @@
+#!/usr/bin/python
+import sys
+import yaml
+import subprocess
+import base64
+
+mtu = int(sys.argv[1]) if len(sys.argv) > 1 else 8900
+print("Setting MTU of %s" % mtu)
+
+ping_size = mtu - 28
+
+udev_rules = """
+SUBSYSTEM=="net", ACTION=="add", KERNEL=="veth*", ATTR{mtu}=="1500", ATTR{mtu}="%d"
+SUBSYSTEM=="net", ACTION=="add", KERNEL=="br-*", ATTR{mtu}=="1500", ATTR{mtu}="%d"
+SUBSYSTEM=="net", ACTION=="add", KERNEL=="gre_system", ATTR{mtu}=="1462", ATTR{mtu}="%d"
+SUBSYSTEM=="net", ACTION=="add", KERNEL=="gre_sys", ATTR{mtu}=="1462", ATTR{mtu}="%d"
+""" % (mtu, mtu, mtu - 38, mtu - 38)
+
+rules_file = '/etc/udev/rules.d/90-bootstack-mtu.rules'
+
+script = """shopt -s nullglob
+    cd /sys/class/net
+    for i in eth? bond? br? en* veth*;do
+      ifconfig $i mtu '%d'
+    done
+    cp -f /etc/network/interfaces /etc/network/interfaces.mtu-bak
+    sed -i\"\" -e\"s/mtu\s*[0-9]*/mtu %d/g\" /etc/network/interfaces
+    echo '%s' |base64 -d > '%s' && udevadm control --reload
+""" % (mtu, mtu, base64.b64encode(udev_rules), rules_file)
+
+juju_status = yaml.load(subprocess.check_output(['juju', 'status', '--format=yaml']))
+
+
+def green_pass():
+    print('\033[92m' + "OK" + '\033[0m')
+
+
+def red_fail():
+    print('\033[91m' + "FAIL" + '\033[0m')  
+
+
+def ping(ip):
+    ping_command = 'ping'
+    if ':' in ip:
+        ping_command = 'ping6'
+    return subprocess.call([ping_command, "-c", "1", "-W", "1", "-s", str(ping_size),
+                            ip], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                           close_fds=True) == 0
+
+
+def ping_result(host, ip):
+    print("Pinging %s (%s) with size %s: " % (host, ip, ping_size)),
+    if ping(ip):
+        green_pass()
+    else:
+        red_fail()
+
+
+def adjust_interfaces(host):
+    print("Setting MTUs on host: %s " % host),
+    if subprocess.call(['juju', 'ssh', host, 'sudo', 'bash', '-c', """'%s'""" % script]) == 0:
+        green_pass()
+    else:
+        red_fail()  
+
+
+for key, machine in juju_status['machines'].iteritems():
+    host = key
+    adjust_interfaces(host)
+    for ip in machine['ip-addresses']:
+        ping_result(host, ip)
+    if 'containers' in machine:
+        for key, container in machine['containers'].iteritems():
+            host = key
+            adjust_interfaces(host)
+            for ip in container['ip-addresses']:
+                ping_result(host, ip)