Merge lp:~1chb1n/ubuntu-openstack-ci/deployer-refactor into lp:ubuntu-openstack-ci

[Ryan Beisner]

refactor the deployer scripts; add initial heat support; mp_comment fixes

* update nova-nothing instance tests
* fix for cli checks
* fix logic error in basic cli test function
* fix touchfile naming for deploy scripts
* add heat and ceilometer basic cli check
* adjust deploy test tempest error detection
* add no-post-destroy
* add heat stack for tempest orchestration tests
* handle mp branch string no-match in deployer script
* add admin script to list or delete ~/images on jenkins nodes
* fix makefile executor target name parsing
* lower mp-comment verbosity for passing tests
* initial prep for multiple tests per mp comment

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #154 lp:ubuntu-openstack-ci for 1chb1n mp243738
    LINT OK: passed

Build: http://10.230.18.80:8080/job/charm_lint_check/154/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #117 lp:ubuntu-openstack-ci for 1chb1n mp243738
    UNIT OK: passed

Build: http://10.230.18.80:8080/job/charm_unit_test/117/

[Ryan Beisner (1chb1n)]

review: Approve

=== added file 'admin/650-boot-nova-nothings.sh'
--- admin/650-boot-nova-nothings.sh	1970-01-01 00:00:00 +0000
+++ admin/650-boot-nova-nothings.sh	2014-12-10 16:14:04 +0000
@@ -0,0 +1,37 @@
+#!/bin/bash -e
+echo $0
+
+# Nova boot a bunch of instances on each OSCI network just for exercise
+
+param=$1
+
+if [ -z "$param" ]; then
+  echo "Potentially dangerous, so you must confirm with YES."
+  echo "Usage:  <this script> YES"
+fi
+
+if [ "$param" != "YES" ]; then
+  exit 0
+fi
+
+. ~/novarc
+
+net_uuids="$(neutron net-list | grep osci | awk '{ print $2 }' || true)"
+if [ -z "$net_uuids" ]; then
+  echo "Couldn't get UUIDs for OSCI neutron networks."
+  exit 1
+fi
+
+img_id="$(nova image-list | grep trusty-daily | tail -n 1 | awk '{ print $2 }' || true)"
+if [ -z "$img_id" ]; then
+  echo "Couldn't get UUID for lastest daily trusty image."
+  exit 1
+fi
+
+# add test key if non-existent
+nova keypair-list | grep testkey &> /dev/null || { nova keypair-add testkey > testkey.pem; chmod 600 testkey.pem; }
+
+# boot stuff
+for net_id in $net_uuids; do
+  nova boot --num-instances 10 --image ${img_id} --flavor m1.smallswap --key_name testkey --nic net-id=${net_id} "osci-nova-nothing"
+done

=== added file 'admin/651-ping-nova-nothings.sh'
--- admin/651-ping-nova-nothings.sh	1970-01-01 00:00:00 +0000
+++ admin/651-ping-nova-nothings.sh	2014-12-10 16:14:04 +0000
@@ -0,0 +1,20 @@
+#!/bin/bash -e
+echo $0
+
+# Ping a bunch of instances on each OSCI network just for exercise
+
+. $(dirname $0)/../env-common
+. ~/novarc
+
+ip_addrs="$(nova list | grep nova-nothing | awk '{ print $12 '} | cut -f 2 -d "=" | sort || true)"
+if [ -z "$ip_addrs" ]; then
+  echo "Couldn't get IP addresses for OSCI nova-nothing instances."
+  exit 1
+fi
+for ip_addr in $ip_addrs; do
+  echo "========== ${ip_addr} =========="
+  echo -n "ping: "
+  ping -c 1 $ip_addr &> /dev/null && echo OK. || echo TROUBLE!
+  echo -n "nc ssh: "
+  nc -w 1 $ip_addr 22 &> /dev/null && echo OK. || echo TROUBLE!
+done

=== added file 'admin/655-delete-nova-nothings.sh'
--- admin/655-delete-nova-nothings.sh	1970-01-01 00:00:00 +0000
+++ admin/655-delete-nova-nothings.sh	2014-12-10 16:14:04 +0000
@@ -0,0 +1,28 @@
+#!/bin/bash -e
+echo $0
+
+# Nova delete any "nova nothing" test instances in the current tenant
+
+param=$1
+
+if [ -z "$param" ]; then
+  echo "Potentially dangerous, so you must confirm with YES."
+  echo "Usage:  <this script> YES"
+fi
+
+if [ "$param" != "YES" ]; then
+  exit 0
+fi
+
+. ~/novarc
+
+instance_ids="$(nova list | grep osci-nova-nothing | awk '{ print $2 }' || true)"
+
+if [ -z "$instance_ids" ]; then
+  echo "Couldn't get UUIDs for OSCI nova-nothing instances."
+  exit 1
+fi
+
+for instance_id in $instance_ids; do
+  nova delete $instance_id
+done

=== added file 'admin/930-images-delete.sh'
--- admin/930-images-delete.sh	1970-01-01 00:00:00 +0000
+++ admin/930-images-delete.sh	2014-12-10 16:14:04 +0000
@@ -0,0 +1,27 @@
+#!/bin/bash -e
+echo $0
+
+# Delete the contents of /var/lib/jenkins/images from the jenkins master and all jenkins slaves (!!)
+
+param=$1
+
+if [ -z "$param" ]; then
+  echo "Deletes the ~/images contents on all jenkins units!"
+  echo "Potentially dangerous, so you must confirm with YES."
+  echo "Usage:  <this script> YES"
+  sleep 2
+fi
+
+# Get jenkins units
+units=$(juju status | egrep "osci[-]?(slave)?/[0-9]{1,10}:" \
+    | cut -d ":" -f 1 | sed 's/^ *//')
+for unit in $units; do
+  echo "----- ${unit} -----"
+  if [ "$param" == "YES" ]; then
+    juju ssh $unit "sudo ls -al /var/lib/jenkins/images && sudo rm -fv /var/lib/jenkins/images/* || echo 'No Images Dir'"
+  else
+    echo "  Not deleting, just listing:"
+    juju ssh $unit "sudo ls -al /var/lib/jenkins/images || echo 'No Images Dir'"
+  fi
+done
+

=== modified file 'job-parts/deploy_with_deployer.sh'
--- job-parts/deploy_with_deployer.sh	2014-12-04 21:43:55 +0000
+++ job-parts/deploy_with_deployer.sh	2014-12-10 16:14:04 +0000
@@ -1,132 +1,61 @@
 #!/bin/bash -e
-# Prep for deploy
-
-${OSCI_ROOT}/job-parts/system_tools_refresh.sh
-${OSCI_ROOT}/job-parts/juju_cleanup.sh
-
-# Gather node info and package info
-env | sort | tee $WORKSPACE/env.$BUILD_NUMBER
-${OSCI_ROOT}/job-parts/get_pkg_info.sh || true
-
-# Handle merge-proposal triggers
-if [[ -n "$MP_TRIGGER" ]] && [[ -n "$MP_SRC_BR" ]] && [[ -n "$MP_TGT_BR" ]] && [[ -n "$DISPLAY_NAME" ]]; then
-  echo "Triggered by merge proposal: ${MP_TRIGGER}"
-  $OSCI_ROOT/job-parts/find_replace.py -yd -i "$OPENSTACK_CHARM_TESTING_ROOT/$CHARM_SET" -o "$WORKSPACE/custom-other.yaml" -f "$MP_TGT_BR" -r "$MP_SRC_BR"
-  export CHARM_SET="custom-other.yaml"
-fi
-
-# Deploy new openstack on openstack environment (tempstack)
-# Handle deployer bundle file upload
-if [[ "$CHARM_SET" == "custom-other.yaml" ]] && [[ -f custom-other.yaml ]]; then
-  echo "Using custom-other deployer bundle yaml file..."
-  BUNDLE_FILE=$WORKSPACE/$CHARM_SET
-  touch $WORKSPACE/fyi.custom.bundle
-else
-  BUNDLE_FILE=$OPENSTACK_CHARM_TESTING_ROOT/$CHARM_SET
-  cp $OPENSTACK_CHARM_TESTING_ROOT/$CHARM_SET $WORKSPACE/$CHARM_SET.$BUILD_NUMBER
-fi
-
-INST_SOURCE=""
-[[ "$INSTALLATION_SOURCE" != "default" ]] && INST_SOURCE="-${INSTALLATION_SOURCE}"
-
-set -x
-(cd $OPENSTACK_CHARM_TESTING_ROOT &&\
-juju-deployer --bootstrap -c ${BUNDLE_FILE} -d ${UBUNTU_RELEASE}-${OPENSTACK_RELEASE}${INST_SOURCE}) \
-  2>&1 | tee $WORKSPACE/deployer.$BUILD_NUMBER
-sleep 20
+# OpenStack Test Deployment
+test="deploy"
+
+# Source the common functions
+. ${OSCI_ROOT}/job-parts/osci_job_common.sh
+
+#f_fresh_clean_start
+# NOTE: workspace not cleaned at this step, as the jenkins job may be copying
+# artifacts to the workspace from another job (such as uploading a custom bundle).
+# Workspace should have been cleaned as step 0 in the jenkins job.
+f_refreshtools_jujuclean
+f_env_no_pass
+f_touch_try
+
+# Handle merge-proposal trigger and substitute the proposed branch in the bundle file
+f_openstack_mergeprop_bundle_handler
+
+# Handle custom deployer bundle file if present
+f_openstack_custom_bundle_handler
+
+# Deploy temp stack
+f_openstack_deploy
 
 # Confirm deployment, configure it, run tempest smoke
+echo "Confirming deployment..."
 juju_ok="$(${OSCI_ROOT}/job-parts/juju_deploy_ok.sh || true)"
 echo $juju_ok
 
 if [[ "$juju_ok" == *DEPLOYED:* ]]; then
-  echo "Juju deployment succeeded."
-  touch $WORKSPACE/fyi.deploy.ok
-
-  . $OPENSTACK_CHARM_TESTING_ROOT/novarc
-  set -x
-  # just cycle to confirm cli is ok
-  neutron net-list 1> /dev/null || echo cli-fail
-  neutron subnet-list 1> /dev/null || echo cli-fail
-  nova network-list 1> /dev/null || echo cli-fail
-  nova keypair-list 1> /dev/null || echo cli-fail
-  nova secgroup-list 1> /dev/null || echo cli-fail
-  keystone user-list 1> /dev/null || echo cli-fail
-  keystone tenant-list 1> /dev/null || echo cli-fail
-  keystone endpoint-list 1> /dev/null || echo cli-fail
-  keystone service-list 1> /dev/null || echo cli-fail
-  glance image-list 1> /dev/null || echo cli-fail
-  nova image-list 1> /dev/null || echo cli-fail
-  cinder list 1> /dev/null || echo cli-fail
-  set +x
-
-  # Configure the deployed tempstack
-  echo "Calling tempstack configure to tweak network, add test keys..."
-  ${OSCI_ROOT}/job-parts/tempstack_configure.sh 2>&1 | tee $WORKSPACE/tempstack-configure.$BUILD_NUMBER || true
-
-  if [[ -n "$(grep testkey.pem $WORKSPACE/tempstack-configure.$BUILD_NUMBER || true)" ]]; then
-    touch $WORKSPACE/fyi.tempstack-configure.ok
-  else
-    echo "Tempstack configure failed."
-    touch $WORKSPACE/fyi.tempstack-configure.bad
-  fi
-
-  cd $OPENSTACK_CHARM_TESTING_ROOT
-  ./configure 2>&1 | tee $WORKSPACE/configure.$BUILD_NUMBER || true
-  cp $OPENSTACK_CHARM_TESTING_ROOT/tempest.conf $WORKSPACE/tempest.conf.$BUILD_NUMBER || true
-
-  if [[ -f $WORKSPACE/tempest.conf.$BUILD_NUMBER ]]; then
-    echo "Configure succeeded."
-    touch $WORKSPACE/fyi.configure.ok
-  else
-    echo "Configure failed."
-    touch $WORKSPACE/fyi.configure.bad
-  fi
-
-  if [[ -n "$(ls $WORKSPACE -1 | egrep '.failed|.bad')" ]]; then
-    echo "Skipping tempest, one or more preceding steps are failed or bad."
-    touch $WORKSPACE/fyi.tempest.skipped
-  else
-    # Kick off tempest
-    (cd $OPENSTACK_CHARM_TESTING_ROOT/tempest &&\
-    sudo pip install -r requirements.txt &&\
-    ./run_tempest.sh -N --smoke) 2>&1 |\
-    tee $WORKSPACE/tempest.$BUILD_NUMBER
-
-    if [[ -n "$(grep 'failures=' $WORKSPACE/tempest.$BUILD_NUMBER)" ]]; then
-      echo "One or more tempest smoke tests failed."
-      touch $WORKSPACE/fyi.tempest.bad
-    elif [[ -z "$(grep 'failures=' $WORKSPACE/tempest.$BUILD_NUMBER)" ]] &&\
-         [[ -n "$(grep 'tempest.scenario.test' $WORKSPACE/tempest.$BUILD_NUMBER)" ]] ; then
-      echo "All tempest smoke tests passed!"
-      touch $WORKSPACE/fyi.tempest.ok
-    else
-      echo "Unknown tempest condition."
-    fi
-  fi
+  echo " . Juju deployment succeeded." && touch $WORKSPACE/fyi-${test:0:4}.deploy.ok
+
+  # Validate CLI authentication and basic functionality
+  f_openstack_cli_basic_check
+
+  # Misc post-deploy configs and checks
+  f_openstack_temp_config
+
+  # Configure via o-c-t script
+  f_openstack_configure
+
+  # Create heat stack from git templates repo
+  f_openstack_heat_stack_create
+
+  # Install tempest requirements
+  f_openstack_tempest_deps
+
+  # Run tempest smoke tests
+  f_openstack_run_tempest_smoke
 else
-  echo "Juju deployment failed."
-  echo "Skipping tempest."
+  echo " ! Juju deployment failed."
+  echo " ! Skipping tempest."
   touch $WORKSPACE/fyi.tempest.skipped
   touch $WORKSPACE/fyi.deploy.failed
 fi
 
-# Collect juju stat and logs
-echo "Collecting juju status and unit logs..."
-. $JENKINS_HOME/novarc
-${OSCI_ROOT}/job-parts/pull_juju_logs.py || true
-${OSCI_ROOT}/job-parts/collect_charm_revno_info.sh || true
-nova list &> $WORKSPACE/nova-list.$BUILD_NUMBER || true
-juju stat &> $WORKSPACE/juju-stat.$BUILD_NUMBER || true
-
-# Build description, combine b_d with fyi. list and echo for regex
-${OSCI_ROOT}/job-parts/b_d.sh || true
-
-# Tear down
-${OSCI_ROOT}/job-parts/juju_cleanup.sh || true
-
-# end
-if [[ "$(ls $WORKSPACE -1)" == *.failed* ]]; then
-  echo "Exiting 1."
-  exit 1
-fi
+f_collect_juju_logs
+f_set_build_descr
+#f_post_mp_comment
+f_nice_finish
+f_openstack_check_fail

=== modified file 'job-parts/lint_check.sh'
--- job-parts/lint_check.sh	2014-10-31 02:35:31 +0000
+++ job-parts/lint_check.sh	2014-12-10 16:14:04 +0000
@@ -4,7 +4,7 @@
 
 # Makefile command search strings, first match wins
 sstring[0]="@flake8"
-#sstring[1]=""
+sstring[1]="/flake8"
 #sstring[2]=""
 #sstring[3]=""
 #sstring[4]=""

=== modified file 'job-parts/osci_job_common.sh'
--- job-parts/osci_job_common.sh	2014-10-31 02:35:31 +0000
+++ job-parts/osci_job_common.sh	2014-12-10 16:14:04 +0000
@@ -8,7 +8,22 @@
 
 function f_fresh_clean_start() {
 # Generally run at the start of every build
+# when no artifacts are copied into the workspace.
+  f_workspace_cleanup
+  f_refreshtools_jujuclean
+}
+
+
+function f_workspace_cleanup() {
+# Purge all files from current workspace
   ${OSCI_ROOT}/job-parts/workspace_cleanup.sh
+}
+
+
+function f_refreshtools_jujuclean() {
+# Generally run at the start of every build
+# when artificats are copied into the workspace from
+# another project.
   ${OSCI_ROOT}/job-parts/system_tools_refresh.sh
   ${OSCI_ROOT}/job-parts/juju_cleanup.sh
 }
@@ -16,7 +31,7 @@
 
 function f_touch_try() {
 # Create touchfile, indicating 'tried to run this test'
-  touch $WORKSPACE/try-${test}
+  touch $WORKSPACE/try-${test:0:4}
 }
 
 
@@ -28,7 +43,9 @@
 
 function f_nice_finish() {
 # Generally run at the end of every build
-  ${OSCI_ROOT}/job-parts/juju_cleanup.sh
+  if [[ "$NO_POST_DESTROY" != "true" ]]; then
+    ${OSCI_ROOT}/job-parts/juju_cleanup.sh
+  fi
 }
 
 
@@ -44,6 +61,19 @@
 }
 
 
+function f_openstack_check_fail() {
+# Check for failure touchfiles and fail if found
+# Exclude '.bad' statuses such as tempest fail
+# ie. pass the deploy even as tempest fails
+  lso="$(ls $WORKSPACE -1)"
+  if [[ "$lso" == *.failed* ]] || \
+     [[ "$lso" == *.missing* ]]; then
+    echo "Exiting 1"
+    exit 1
+  fi
+}
+
+
 function f_post_mp_comment() {
 # Post mp comment if triggered by a merge proposal
   if [[ -n "$MP_TRIGGER" ]]; then
@@ -71,7 +101,7 @@
 # check out branch
   if [[ -z "$BASE_NAME" ]] || [[ -z "$BRANCH" ]]; then
     echo " ! No BRANCH and/or BASE_NAME env vars."
-    touch $WORKSPACE/fyi-${test}.env-vars.bad
+    touch $WORKSPACE/fyi-${test:0:4}.env-vars.bad
   else
     mkdir -pv $JENKINS_HOME/checkout
     CO_DIR=$JENKINS_HOME/checkout/$BASE_NAME
@@ -93,6 +123,7 @@
 function f_run_makefile_test() {
 # Run makefile-based test (generally lint, unit or amulet)
   cd $CO_DIR
+  # sstring array set by job
   q_sstring_l=( "${sstring[@]/#/\"}" )
   q_sstring=( "${q_sstring_l[@]/%/\"}" )
   echo " . Running test from Makefile."
@@ -118,3 +149,157 @@
       ;;
   esac
 }
+
+
+function f_collect_juju_logs() {
+# Collect juju stat and logs
+  echo "Collecting juju status and unit logs..."
+  . $JENKINS_HOME/novarc
+  ${OSCI_ROOT}/job-parts/pull_juju_logs.py || true
+  ${OSCI_ROOT}/job-parts/collect_charm_revno_info.sh || true
+  nova list &> $WORKSPACE/nova-list.$BUILD_NUMBER || true
+  juju stat &> $WORKSPACE/juju-stat.$BUILD_NUMBER || true
+}
+
+
+function f_openstack_mergeprop_bundle_handler() {
+# Handle merge-proposal trigger and substitute the proposed branch in the bundle file
+  if [[ -n "$MP_TRIGGER" ]] && [[ -n "$MP_SRC_BR" ]] && [[ -n "$MP_TGT_BR" ]] && [[ -n "$DISPLAY_NAME" ]]; then
+    echo "Triggered by merge proposal: ${MP_TRIGGER}"
+    echo "Substituting proposed branch in deployer bundle..."
+    echo '' > $WORKSPACE/custom-other.yaml || true
+    $OSCI_ROOT/job-parts/find_replace.py -yd -i "$OPENSTACK_CHARM_TESTING_ROOT/$CHARM_SET" -o "$WORKSPACE/custom-other.yaml" -f "$MP_TGT_BR" -r "$MP_SRC_BR" || touch $WORKSPACE/fyi-${test:0:4}.mp-branch-replace.failed
+    export CHARM_SET="custom-other.yaml"
+  fi
+}
+
+
+function f_openstack_custom_bundle_handler() {
+# Handle custom deployer bundle file if present
+  if [[ "$CHARM_SET" == "custom-other.yaml" ]] && [[ -f custom-other.yaml ]]; then
+    echo "Using custom-other deployer bundle yaml file..."
+    BUNDLE_FILE=$WORKSPACE/$CHARM_SET
+    touch $WORKSPACE/fyi-${test:0:4}.custom.bundle
+  else
+    BUNDLE_FILE=$OPENSTACK_CHARM_TESTING_ROOT/$CHARM_SET
+    cp $OPENSTACK_CHARM_TESTING_ROOT/$CHARM_SET $WORKSPACE/$CHARM_SET.$BUILD_NUMBER
+  fi
+}
+
+
+function f_openstack_deploy() {
+# Deploy the stack
+  echo "Deploying OpenStack..."
+  INST_SOURCE=""
+  [[ "$INSTALLATION_SOURCE" != "default" ]] && INST_SOURCE="-${INSTALLATION_SOURCE}"
+  set -x
+  (cd $OPENSTACK_CHARM_TESTING_ROOT &&\
+  juju-deployer --bootstrap -c ${BUNDLE_FILE} -d ${UBUNTU_RELEASE}-${OPENSTACK_RELEASE}${INST_SOURCE}) \
+    2>&1 | tee $WORKSPACE/deployer.$BUILD_NUMBER
+  set +x
+  sleep 30
+}
+
+
+function f_openstack_temp_config() {
+# Configure the deployed tempstack
+  echo "Calling tempstack configure to tweak network, add test keys..."
+  ${OSCI_ROOT}/job-parts/tempstack_configure.sh 2>&1 | tee $WORKSPACE/tempstack-configure.$BUILD_NUMBER || true
+
+  if [[ -n "$(grep "END RSA PRIVATE KEY" $WORKSPACE/testkey.pem.$BUILD_NUMBER || true)" ]]; then
+    echo " . Tempstack configure ok."
+    touch $WORKSPACE/fyi-${test:0:4}.tempstack-configure.ok
+  else
+    echo " ! Tempstack configure failed."
+    touch $WORKSPACE/fyi-${test:0:4}.tempstack-configure.bad
+  fi
+}
+
+
+function f_openstack_configure() {
+# Run the openstack-charm-testing ./configure script
+  echo "Configuring..."
+  cd $OPENSTACK_CHARM_TESTING_ROOT
+  ./configure 2>&1 | tee $WORKSPACE/configure.$BUILD_NUMBER || true
+  cp $OPENSTACK_CHARM_TESTING_ROOT/tempest.conf $WORKSPACE/tempest.conf.$BUILD_NUMBER || true
+
+  if [[ -f $WORKSPACE/tempest.conf.$BUILD_NUMBER ]]; then
+    echo " . Configure succeeded."
+    touch $WORKSPACE/fyi-${test:0:4}.configure.ok
+  else
+    echo " ! Configure failed."
+    touch $WORKSPACE/fyi-${test:0:4}.configure.bad
+  fi
+}
+
+
+function f_openstack_heat_stack_create() {
+# add a basic heat HOT template stack for tempest orchestration tests
+# experimental, force pass for now
+  heat stack-create heat_basic -f $OPENSTACK_CHARM_TESTING_ROOT/heat-templates/heat-basic.yaml \
+    --parameters "admin_pass=Ubuntu;key_name=testkey;image=cirros;network=private" || true
+  heat stack-list || true
+}
+
+
+function f_openstack_tempest_deps() {
+# Install tempest requirements
+  echo "Installing tempest requirements..."
+  (cd $OPENSTACK_CHARM_TESTING_ROOT/tempest &&\
+  sudo pip install -r requirements.txt) 2>&1 |\
+  tee $WORKSPACE/tempest-deps.$BUILD_NUMBER
+}
+
+
+function f_openstack_run_tempest_smoke() {
+# Run tempest if nothing has failed so far
+  if [[ -n "$(ls $WORKSPACE -1 | egrep '.failed|.bad')" ]]; then
+    echo " ! Skipping tempest, one or more preceding steps are failed or bad."
+    touch $WORKSPACE/fyi-${test:0:4}.tempest.skipped
+  else
+    # Kick off tempest
+    (cd $OPENSTACK_CHARM_TESTING_ROOT/tempest && ./run_tempest.sh -N --smoke) 2>&1 |\
+      tee $WORKSPACE/tempest.$BUILD_NUMBER
+    if [[ -n "$(grep 'failures=' $WORKSPACE/tempest.$BUILD_NUMBER)" ]]; then
+      echo " ! One or more tempest smoke tests failed."
+      touch $WORKSPACE/fyi-${test:0:4}.tempest.bad
+    elif [[ -z "$(grep 'failures=' $WORKSPACE/tempest.$BUILD_NUMBER)" ]] &&\
+         [[ -n "$(grep 'tempest.scenario.test' $WORKSPACE/tempest.$BUILD_NUMBER)" ]] ; then
+      echo " . All tempest smoke tests passed!"
+      touch $WORKSPACE/fyi-${test:0:4}.tempest.ok
+    else
+      echo " ! Unknown tempest result condition."
+    fi
+  fi
+}
+
+
+function f_openstack_cli_basic_check() {
+# Perform basic command line checks against the cloud
+  echo "Confirming basic CLI functionality and authentication..."
+  . $OPENSTACK_CHARM_TESTING_ROOT/novarc
+  tmp_cmds="$(mktemp)"
+  cat > "$tmp_cmds" << "--EOF"
+  neutron net-list
+  neutron subnet-list
+  nova network-list
+  nova keypair-list
+  nova secgroup-list
+  glance image-list
+  nova image-list
+  cinder list
+  swift list
+  keystone user-list
+  keystone tenant-list
+  keystone endpoint-list
+  keystone service-list
+  heat stack-list
+  ceilometer resource-list
+--EOF
+  while read cmd; do
+    echo -n "${cmd}: "
+    $cmd &> /dev/null && echo 'ok' ${PIPESTATUS[0]} || { echo 'cli-fail' ${PIPESTATUS[0]}; touch $WORKSPACE/fyi-${test:0:4}.cli.failed; }
+  done < $tmp_cmds
+  rm -fv $tmp_cmds
+}
+

=== modified file 'job-parts/workspace_cleanup.sh'
--- job-parts/workspace_cleanup.sh	2014-11-26 15:07:38 +0000
+++ job-parts/workspace_cleanup.sh	2014-12-10 16:14:04 +0000
@@ -16,4 +16,3 @@
 sudo cp -fv $OSCI_ROOT/deployment/deploy-files/launchpad_host_keys /var/lib/jenkins/.ssh/known_hosts
 sudo chmod 600 /var/lib/jenkins/.ssh/known_hosts
 sudo chown jenkins:jenkins /var/lib/jenkins/.ssh/known_hosts
-

=== modified file 'make_exec.py'
--- make_exec.py	2014-10-31 15:52:31 +0000
+++ make_exec.py	2014-12-10 16:14:04 +0000
@@ -62,7 +62,7 @@
         if len(_line) != 0 and _line[0] != '#':
             if _this_line[0] != '\t' and ':' in _line:
                 # Makefile target name found
-                _target = _line[:-1]
+                _target = _line.split(':')[0]
                 _cmds = []
                 found_target = True
             elif _this_line[0] == '\t':

=== modified file 'mp_comment.py'
--- mp_comment.py	2014-10-31 17:27:35 +0000
+++ mp_comment.py	2014-12-10 16:14:04 +0000
@@ -140,6 +140,7 @@
     results = []
     pbs = {}
     something_ran = False
+    trigger_tests = []
     for trigger_test, tt_attrs in conf_map['mp-trigger-tests'].iteritems():
         # look for try touchfiles
         try_files = []
@@ -149,14 +150,41 @@
             try_files.extend(these_try_files)
         if len(try_files) > 0:
             something_ran = True
+            trigger_tests.append(trigger_test)
             logging.info('Found {} try file(s)'.format(trigger_test))
         else:
             # did not run
             logging.info('No {} try file(s) found'.format(trigger_test))
             continue
 
+        # look for fail touchfiles
+        logging.debug('Checking for {} fyi files'.format(trigger_test))
+        passed = True
+        for fail_mask in tt_attrs['fail-mask']:
+            logging.debug('Checking fail mask: {}'.format(fail_mask))
+            fyis = glob(os.path.join(WORKSPACE, fail_mask))
+            if len(fyis) > 0:
+                passed = False
+                logging.debug('Found {} fyi file(s)'.format(trigger_test))
+                # handle fyi files
+                for fyi in fyis:
+                    _line = '    {} FAIL: {}'.format(
+                        trigger_test.upper(),
+                        os.path.split(fyi)[1][8:].replace('.', ' '))
+                    logging.debug(_line)
+                    summary.append(_line)
+            else:
+                logging.debug('No "fyi" files found to '
+                              'include for mask: {}'.format(fail_mask))
+
+        # ran and didn't fail
+        if passed:
+            _line = ('    {} OK: passed'.format(trigger_test.upper()))
+            summary.append(_line)
+
         # look for result files
         res_files = []
+        this_result = []
         logging.debug('Checking result mask for {}'.format(trigger_test))
         for res_mask in tt_attrs['result-mask']:
             these_res_files = glob(os.path.join(WORKSPACE, res_mask))
@@ -171,43 +199,23 @@
                 llimit = conf_map['mp-conf']['line-limit']
                 _line = ('\n{} Results (max last {} lines):'.format(
                     trigger_test.upper(), llimit))
-                results.append(_line)
+                this_result.append(_line)
                 with open(res_file) as _file:
                     res_lines = [f_line.decode('ascii', 'ignore').rstrip()
                                  for f_line in _file][-llimit:]
                 _line = ('\nFull {} test output: {}'.format(trigger_test,
                                                             pbs[res_file]))
                 res_lines.append(_line)
-                results.extend(res_lines)
+                this_result.extend(res_lines)
         else:
             _line = ('\n{} Results not found.'.format(trigger_test.upper()))
             results.append(_line)
             logging.info('No {} results files found to '
                          'include'.format(trigger_test))
 
-        # look for fail touchfiles
-        logging.debug('Checking for {} fyi files'.format(trigger_test))
-        passed = True
-        for fail_mask in tt_attrs['fail-mask']:
-            logging.debug('Checking fail mask: {}'.format(fail_mask))
-            fyis = glob(os.path.join(WORKSPACE, fail_mask))
-            if len(fyis) > 0:
-                passed = False
-                logging.debug('Found {} fyi file(s)'.format(trigger_test))
-                # handle fyi files
-                for fyi in fyis:
-                    _line = '    {} FAIL: {}'.format(
-                        trigger_test.upper(),
-                        os.path.split(fyi)[1][8:].replace('.', ' '))
-                    logging.debug(_line)
-                    summary.append(_line)
-            else:
-                logging.debug('No "fyi" files found to '
-                              'include for mask: {}'.format(fail_mask))
-        # ran and didn't fail
-        if passed:
-            _line = ('    {} OK: passed'.format(trigger_test.upper()))
-            summary.append(_line)
+        if not passed:
+            results.extend(this_result)
+
     logging.debug('pbs: {}'.format(pbs))
 
     if not something_ran:
@@ -216,6 +224,7 @@
 
     # add summary and results to context for template
     contexts[0]['summary'] = '\n'.join(summary)
+#    if not passed:
     contexts[0]['results'] = '\n'.join(results)
 
     # create mp comment file
@@ -227,7 +236,8 @@
                                               params=params)
 
     # read mp comment file and display to console
-    subject = '[UOSCI] test result for {}'.format(DISPLAY_NAME)
+    subject = '[UOSCI] {} test result for {}'.format('/'.join(trigger_tests),
+                                                     DISPLAY_NAME)
     logging.debug('Subject: {}'.format(subject))
     f = open(comment_filename, 'r')
     comment = f.read()
@@ -249,10 +259,6 @@
         lp_branch_url = MP_TGT_BR
         mp_url = MP_TRIGGER
 
-    # post mp comment
-    logging.debug('Connecting to LP via API')
-    launchpad = osci_utils.lpl_connect(anonymous=False)
-
     if MPC_SILENT:
         # useful for setting environment variable in a jenkins staging
         # area to prevent mp comment posting globally
@@ -260,6 +266,9 @@
                      'will actually be posted')
 
     if not opts.dry and not opts.no_post and not MPC_SILENT:
+        # post mp comment
+        logging.debug('Connecting to LP via API')
+        launchpad = osci_utils.lpl_connect(anonymous=False)
         logging.debug('Looking up MP object from LP API')
         try:
             mp = osci_utils.lpl_get_mp(launchpad, mp_url, lp_branch_url)

=== modified file 'populate/mappings.yaml'
--- populate/mappings.yaml	2014-11-22 16:16:10 +0000
+++ populate/mappings.yaml	2014-12-10 16:14:04 +0000
@@ -14,7 +14,7 @@
   mp-state-file: 'mp-state'
 
 mp-conf:
-  line-limit: 5
+  line-limit: 2
 # when True, post ALL MP comments to this alternate MP when mp-comment-sandbox
 #   ie. contain the noise during osci development
   mp-comment-sandbox: False

=== modified file 'populate/mp-comment.tmpl'
--- populate/mp-comment.tmpl	2014-10-20 21:05:18 +0000
+++ populate/mp-comment.tmpl	2014-12-10 16:14:04 +0000
@@ -1,4 +1,3 @@
-UOSCI bot says:
 {{JOB_NAME}} {{BUILD_DISPLAY_NAME}}
 {{summary}}
 {{results}}