Merge lp:~dweaver/orange-box/orange-box-status-all-nodes-improvement into lp:orange-box

[Darryl Weaver]

Improve orange-box-status-all-nodes handling and error reporting.
Disable orange-box-amt-monitor service for MAAS 1.7 environments to reduce chances of AMT lockup.

=== removed file 'etc/init/orange-box-amt-monitor.conf'
--- etc/init/orange-box-amt-monitor.conf	2014-04-16 22:26:52 +0000
+++ etc/init/orange-box-amt-monitor.conf	1970-01-01 00:00:00 +0000
@@ -1,11 +0,0 @@
-# orange-box-amt-monitor - monitor the AMT port
-
-description	"Orange Box AMT Monitor"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-respawn
-respawn limit 10 5
-
-exec orange-box-amt-monitor

=== modified file 'usr/bin/orange-box-status-all-nodes'
--- usr/bin/orange-box-status-all-nodes	2014-10-10 08:46:06 +0000
+++ usr/bin/orange-box-status-all-nodes	2014-10-23 22:21:21 +0000
@@ -4,6 +4,7 @@
 #    Copyright (C) 2014 Canonical Ltd.
 #
 #    Authors: Ronald McCollam <[e-mail]>
+#             Darryl Weaver <[e-mail]>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
@@ -22,96 +23,100 @@
 ERRORS=0
 
 okay() {
-	echo "OK:    $@"
+        echo "OK:    $@"
 }
 
 error() {
-	echo "ERROR: $@"
-	ERRORS=$((ERRORS+1))
+        echo "ERROR: $@"
+        ERRORS=$((ERRORS+1))
 }
 
 
 status_all_nodes() {
-	# Check all VM nodes
-	for i in $(seq 0 2); do
-		if virsh list --all | grep node0vm${i} | grep running >/dev/null 2>&1; then
-			okay "[node0vm${i}]: Is running"
-			uptime=$(ssh node0vm${i}.maas "uptime" 2>/dev/null)
-			if [ -n "$uptime" ]; then
-				okay "[node0vm${i}]:  SSH to node successful; uptime=[$uptime]"
-			else
-				error "[node0vm${i}]:  SSH to node unsuccessful; uptime=[$uptime]"
-			fi
-		else
-			error "[node0vm${i}]: Is not running"
-		fi
-	done
-	# Check all AMT nodes
-	for i in $(seq 1 9); do
-		hostname="node${i}.maas"
-		# (1) Ping AMT IP Addresses
-		amt_ip="10.14.4.1$i"
-		if ping -c 1 -q $amt_ip >/dev/null 2>&1; then
-			okay "[$hostname]: AMT responding to ping [$amt_ip]"
-		else
-			error "[$hostname]: AMT not responding to ping [$amt_ip]"
-			continue
-		fi
-		# (2) If AMT pingable, check for AMT open port 16992
-		if netcat -z -v $amt_ip 16992 >/dev/null 2>&1; then
-			okay "[$hostname]:  AMT listening on port [16992] [$amt_ip]"
-		else
-			error "[$hostname]:  AMT not listening on port [16992] [$amt_ip]"
-			continue
-		fi
-		amtoutput=$(yes | amttool $amt_ip info)
-		amtpowerstate=$(echo "$amtoutput" | grep "^Powerstate:" | awk '{print $2}')
-		# (3) If AMT port open, check AMT for power state
-		if [ -n "$amtpowerstate" ]; then
-			okay "[$hostname]:   AMT powerstate [$amtpowerstate] on [$amt_ip]"
-		else
-			error "[$hostname]:   AMT powerstate unknown [$amtpowerstate] on [$amt_ip]"
-			continue
-		fi
-		# (4) If power state is running, test pingable OS
-		ip=$(host $hostname | tail -n1  | sed -e "s/.* //")
-		if ping -c 1 -q $hostname >/dev/null 2>&1; then
-			okay "[$hostname]:    OS responding to ping, ip=[$ip]"
-		else
-			error "[$hostname]:   OS not responding to ping, ip=[$ip]"
-			continue
-		fi
-		uptime=$(ssh $hostname "uptime" 2>/dev/null)
-		# (5) If OS is pingable, test SSH to OS
-		if [ -n "$uptime" ]; then
-			okay "[$hostname]:     SSH to node successful; uptime=[$uptime]"
-		else
-			error "[$hostname]:    SSH to node unsuccessful; uptime=[$uptime]"
-			continue
-		fi
-		memory=$(ssh $hostname "grep -m1 '^MemTotal:' /proc/meminfo | sed -e 's/.*:\s\+//'" 2>/dev/null)
-		# (6) If SSH works, test memory
-		if [ -n "$memory" ]; then
-			okay "[$hostname]:     Memory check of node successful; memory=[$memory]"
-		else
-			error "[$hostname]:    Memory check of node unsuccessful; memory=[$memory]"
-			continue
-		fi
-		# (7) If SSH works, test disk
-		disk=$(ssh $hostname "sudo fdisk -l | grep '^Disk /' | sed -e 's/^Disk //' -e 's/,.*//'" 2>/dev/null)
-		if [ -n "$disk" ]; then
-			okay "[$hostname]:     Disk check of node successful; disk=[$(echo $disk | tr '\n' ' ')]"
-		else
-			error "[$hostname]:    Disk check of node unsuccessful; disk=[$disk]"
-			continue
-		fi
-	done
+        # Check all VM nodes
+        for i in $(seq 0 2); do
+                if virsh list --all | grep node0vm${i} | grep running >/dev/null 2>&1; then
+                        okay "[node0vm${i}]: Is running"
+                        uptime=$(ssh node0vm${i}.maas "uptime" 2>/dev/null)
+                        if [ -n "$uptime" ]; then
+                                okay "[node0vm${i}]:  SSH to node successful; uptime=[$uptime]"
+                        else
+                                error "[node0vm${i}]:  SSH to node unsuccessful; uptime=[$uptime]"
+                        fi
+                else
+                        error "[node0vm${i}]: Is not running"
+                fi
+        done
+        # Check all AMT nodes
+        for i in $(seq 1 9); do
+                hostname="node${i}.maas"
+                # (1) Ping AMT IP Addresses
+                amt_ip="10.14.4.1$i"
+                if ping -c 1 -q $amt_ip >/dev/null 2>&1; then
+                        okay "[$hostname]: AMT responding to ping [$amt_ip]"
+                else
+                        error "[$hostname]: AMT not responding to ping [$amt_ip]"
+                        continue
+                fi
+                # (2) If AMT pingable, check for AMT open port 16992
+                if netcat -z -v $amt_ip 16992 >/dev/null 2>&1; then
+                        okay "[$hostname]:  AMT listening on port [16992] [$amt_ip]"
+                else
+                        error "[$hostname]:  AMT not listening on port [16992] [$amt_ip]"
+                        continue
+                fi
+                amtoutput=$(yes | amttool $amt_ip info)
+                amtpowerstate=$(echo "$amtoutput" | grep "^Powerstate:" | awk '{print $2}')
+                # (3) If AMT port open, check AMT for power state
+                if [ "$amtpowerstate" = "S0" ]; then
+                        okay "[$hostname]:   AMT powerstate [$amtpowerstate](running) on [$amt_ip]"
+                elif [ "$amtpowerstate" = "S5" ]; then
+                        okay "[$hostname]:   AMT powerstate [$amtpowerstate](not running) on [$amt_ip]"
+                        continue
+                else
+                        error "[$hostname]:   AMT powerstate unknown [$amtpowerstate] on [$amt_ip]"
+                        continue
+                fi
+                # (4) If power state is running, test pingable OS
+                ip=$(host $hostname | tail -n1  | sed -e "s/.* //")
+                if ping -c 1 -q $hostname >/dev/null 2>&1; then
+                        okay "[$hostname]:    OS responding to ping, ip=[$ip]"
+                else
+                        error "[$hostname]:   OS not responding to ping, ip=[$ip]"
+                        continue
+                fi
+                uptime=$(ssh $hostname "uptime" 2>/dev/null)
+                # (5) If OS is pingable, test SSH to OS
+                if [ -n "$uptime" ]; then
+                        okay "[$hostname]:     SSH to node successful; uptime=[$uptime]"
+                else
+                        error "[$hostname]:    SSH to node unsuccessful; uptime=[$uptime]"
+                        continue
+                fi
+                memory=$(ssh $hostname "grep -m1 '^MemTotal:' /proc/meminfo | sed -e 's/.*:\s\+//'" 2>/dev/null)
+                # (6) If SSH works, test memory
+                if [ -n "$memory" ]; then
+                        okay "[$hostname]:     Memory check of node successful; memory=[$memory]"
+                else
+                        error "[$hostname]:    Memory check of node unsuccessful; memory=[$memory]"
+                        continue
+                fi
+                # (7) If SSH works, test disk
+                disk=$(ssh $hostname "sudo fdisk -l | grep '^Disk /' | sed -e 's/^Disk //' -e 's/,.*//'" 2>/dev/null)
+                if [ -n "$disk" ]; then
+                        okay "[$hostname]:     Disk check of node successful; disk=[$(echo $disk | tr '\n' ' ')]"
+                else
+                        error "[$hostname]:    Disk check of node unsuccessful; disk=[$disk]"
+                        continue
+                fi
+        done
 }
 
 status_all_nodes | ccze -A
 
 if [ $ERRORS -eq 0 ]; then
-	exit 0
+        exit 0
 else
-	exit $ERRORS
+        exit $ERRORS
 fi
+