Merge lp:~chad.smith/charms/precise/swift-storage/swift-storage-with-health into lp:~charmers/charms/precise/swift-storage/trunk

[Chad Smith]

Add run-parts health_scripts.d directory to be used by landscape-client during rolling OpenStack upgrades to validate the health of swift services after a system reboot. These health scripts use the same mechanism as other openstack charms to create a scriptrc file which will define OPENSTACK_* environment variables used by health scripts to validate various swift services and ports are active on the swift-storage unit. Landscape-client will use run-parts to validate all health scripts exit zero as a validation that any known swift services are in place and running appropriately before upgrading any other swift-storage nodes.

[Chad Smith (chad.smith)]

From Adam (Merged):
Retargetted and merged against lp:~openstack-charmers/charms/precise/swift-storage/ha-support which contains the most recent common code and will let us track this against the other WIP charms focusing on these features.

=== added symlink 'hooks/config-changed'
=== target is u'swift-storage-node-relations'
=== modified file 'hooks/lib/openstack-common'
--- hooks/lib/openstack-common	2013-01-11 18:33:34 +0000
+++ hooks/lib/openstack-common	2013-03-17 15:25:24 +0000
@@ -314,3 +314,42 @@
   echo "$found"
   return 0
 }
+
+##########################################################################
+# Description: Creates an rc file exporting environment variables to a
+# script_path local to the charm's installed directory.
+# Any charm scripts run outside the juju hook environment can source this
+# scriptrc to obtain updated config information necessary to perform health
+# checks or service changes
+#
+# Parameters: 
+#   An array of '=' delimited  ENV_VAR:value combinations to export.
+#   If optional script_path key is not provided in the array, script_path
+#     defaults to scripts/scriptrc
+##########################################################################
+function save_script_rc {
+  if [ ! -n "$JUJU_UNIT_NAME" ]; then
+     echo "Error: Missing JUJU_UNIT_NAME environment variable"
+     exit 1
+  fi
+  # our default unit_path
+  unit_path="/var/lib/juju/units/${JUJU_UNIT_NAME/\//-}/charm/scripts/scriptrc"
+  echo $unit_path
+  tmp_rc="/tmp/${JUJU_UNIT_NAME/\//-}rc"
+
+  echo "#!/bin/bash" > $tmp_rc
+  for env_var in "${@}"
+  do
+    if `echo $env_var | grep -q script_path`; then
+       # well then we need to reset the new unit-local script path
+       unit_path="/var/lib/juju/units/${JUJU_UNIT_NAME/\//-}/charm/${env_var/script_path=/}"
+    else
+       echo "export $env_var" >> $tmp_rc
+    fi 
+  done
+  chmod 755 $tmp_rc
+  mv $tmp_rc $unit_path
+}
+
+
+

=== modified file 'hooks/swift-storage-node-relations'
--- hooks/swift-storage-node-relations	2013-01-15 18:40:49 +0000
+++ hooks/swift-storage-node-relations	2013-03-17 15:25:24 +0000
@@ -11,6 +11,26 @@
   echo "ERROR: Could not load swift-storage-node-common from $CHARM_DIR"
 fi
 
+function config_changed {
+  
+  declare -a env_vars=()
+  for i in account container object ; do
+    port=$(config-get ${i}-server-port)
+    local url="http://$STORAGE_LOCAL_NET_IP:$port/recon/diskusage"
+    # append to env_vars
+    env_vars+=("OPENSTACK_PORT_${i^^}=$port")
+    env_vars+=("OPENSTACK_SWIFT_SERVICE_${i^^}=${i}-server")
+
+    # Ensure we have at least one device mounted as reported by swift-recon
+    env_vars+=('OPENSTACK_URL_'${i^^}'="'$url'|\"mounted\":+\"true\""')
+
+    create_server_conf $i "$port"
+  done
+
+  # Save our scriptrc env variables for health checks
+  save_script_rc ${env_vars[@]}
+} 
+
 function install_hook {
   apt-get -y --force-yes install python-software-properties || exit 1
 
@@ -27,10 +47,7 @@
   configure_rsyncd
   swift-init all stop || true
   setup_storage
-  for i in account container object ; do
-    port=$(config-get ${i}-server-port)
-    create_server_conf $i "$port"
-  done
+  config_changed
 }
 
 function storage_joined {
@@ -70,6 +87,7 @@
 case $ARG0 in
   "install") install_hook ;;
   "start"|"stop") exit 0 ;;
+  "config-changed") config_changed ;;
   "swift-storage-relation-joined") storage_joined ;;
   "swift-storage-relation-changed") storage_changed ;;
 esac

=== modified file 'revision'
--- revision	2013-01-11 18:33:34 +0000
+++ revision	2013-03-17 15:25:24 +0000
@@ -1,1 +1,1 @@
-53
+54

=== added directory 'scripts'
=== added directory 'scripts/health_checks.d'
=== added file 'scripts/health_checks.d/service_ports_live'
--- scripts/health_checks.d/service_ports_live	1970-01-01 00:00:00 +0000
+++ scripts/health_checks.d/service_ports_live	2013-03-17 15:25:24 +0000
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Validate that service ports are active
+HEALTH_DIR=`dirname $0`
+SCRIPTS_DIR=`dirname $HEALTH_DIR`
+. $SCRIPTS_DIR/scriptrc
+set -e
+
+# Grab any OPENSTACK_PORT* environment variables
+openstack_ports=`env| awk -F '=' '(/OPENSTACK_PORT/){print $2}'`
+for port in $openstack_ports
+do
+    netstat -ln | grep -q $port
+done

=== added file 'scripts/health_checks.d/service_swift_running'
--- scripts/health_checks.d/service_swift_running	1970-01-01 00:00:00 +0000
+++ scripts/health_checks.d/service_swift_running	2013-03-17 15:25:24 +0000
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Validate that service is running
+HEALTH_DIR=`dirname $0`
+SCRIPTS_DIR=`dirname $HEALTH_DIR`
+. $SCRIPTS_DIR/scriptrc
+set -e
+
+# Grab any OPENSTACK_SWIFT_SERVICE* environment variables
+openstack_service_names=`env| awk -F '=' '(/OPENSTACK_SWIFT_SERVICE/){print $2}'`
+for service_name in $openstack_service_names
+do
+    # Double-negative: we want to ensure swift-init does not return
+    # 'No <service_name> running'
+    swift-init $service_name status 2>/dev/null | grep -vq "No $service_name running"
+done

=== added file 'scripts/health_checks.d/service_url_checks'
--- scripts/health_checks.d/service_url_checks	1970-01-01 00:00:00 +0000
+++ scripts/health_checks.d/service_url_checks	2013-03-17 15:25:24 +0000
@@ -0,0 +1,16 @@
+#!/bin/bash
+# Validate that service urls return expected content
+HEALTH_DIR=`dirname $0`
+SCRIPTS_DIR=`dirname $HEALTH_DIR`
+. $SCRIPTS_DIR/scriptrc
+set -e
+
+# Grab any OPENSTACK_URL* environment variables and validate content response
+openstack_urls=`env| awk -F '=' '(/OPENSTACK_URL/){print $2 }'`
+for url_check in $openstack_urls
+do
+    url=`echo $url_check| awk -F '|' '{print $1}'`
+    expected_content=`echo $url_check| awk -F '|' '{print $2}'`
+    wget -q -O - $url | grep -q "${expected_content//+/ }"
+done
+