Merge lp:~thomir-deactivatedaccount/adt-cloud-worker/uci-nova-proxy into lp:~canonical-ci-engineering/adt-cloud-worker/uci-nova

[Thomi Richards]

Add support for proxy server.

Support --proxy=<proxy_url> which will endup in the test environment http[s]_proxy variables.

This option should not be set to "squid.internal" until it is accessible and fully functional, otherwise adt-run won't be able to fetch sources and perform other network operations (yes, everything in the test will be proxied, except cloud-init).

Also silencing more neutron calls as a drive-by cleanup.

[Francis Ginther (fginther)]

Re-merge approved.

review: Approve

=== modified file 'uci-nova'
--- uci-nova	2015-04-27 18:14:06 +0000
+++ uci-nova	2015-04-28 02:24:43 +0000
@@ -19,6 +19,8 @@
 #        Name for the new server. A name will be generated if not specified.
 # -m mirror | --mirror=mirror
 #        Optional ubuntu archive mirror to override the default.
+# -p proxy_url | --proxy=proxy_url
+#        Optional HTTP[S] proxy url to populate testbed environment.
 # -c console-log | --console=file-name
 #        Save the nova console-log of the server to the specified file.
 #
@@ -59,6 +61,7 @@
 NET_ID=""
 CONSOLE=""
 MIRROR=""
+PROXY=""
 DEBUG=""
 
 
@@ -78,8 +81,8 @@
 parse_args() {
     # Parse command line argument and populate environment
 
-    SHORTOPTS="f:,i:,N:,n:,m:,c:,d"
-    LONGOPTS="flavor:,image:,net-id:,name:,mirror:,console:,debug"
+    SHORTOPTS="f:,i:,N:,n:,m:,p:,c:,d"
+    LONGOPTS="flavor:,image:,net-id:,name:,mirror:,proxy:,console:,debug"
 
     TEMP=$(getopt -o $SHORTOPTS --long $LONGOPTS -- "$@")
     eval set -- "$TEMP"
@@ -101,6 +104,9 @@
             -m|--mirror)
                 MIRROR=$2
                 shift 2;;
+            -p|--proxy)
+                PROXY=$2
+                shift 2;;
             -c|--console)
                 CONSOLE=$2
                 shift 2;;
@@ -173,6 +179,21 @@
     	--direction egress \
     	--remote-ip-prefix 91.189.88.0/21 \
     	$SRVNAME
+
+    if [ -n "$PROXY" ]; then
+        debug "Allowing internet proxy egress traffic ..."
+        proxy_hostname=$(echo $PROXY | sed -e "s/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/")
+        if [ $(echo "$proxy_hostname" | grep -E "^[0-9]{1,3}(\.[0-9]{1,3}){3}$") ]; then
+            proxy_ip=$proxy_hostname
+        else 
+            proxy_ip=$(host $proxy_hostname | cut -d' ' -f4)
+        fi
+
+        neutron security-group-rule-create \
+            --direction egress \
+            --remote-ip-prefix $proxy_ip/24 \
+            $SRVNAME
+    fi
 }
 
 # create a testbed (if necessary), configure ssh, copy ssh key into it,
@@ -193,7 +214,7 @@
 
     # Setup testbed security with nova or neutron according to their
     # availability in the target cloud.
-    if type neutron >/dev/null && neutron security-group-list >/dev/null; then
+    if type neutron >/dev/null && neutron security-group-list >/dev/null 2>&1; then
         debug "Setting up neutron secgroup"
         security_setup_neutron
     else
@@ -220,6 +241,12 @@
     # from the performance perspective.
     swap_setup="fallocate -l 4G /swapfile; chmod 600 /swapfile; mkswap /swapfile; swapon /swapfile; echo \"/swapfile none swap sw 0 0\" >> /etc/fstab"
 
+    # Testbed environment setup to be executed by cloud-init.
+    # Sets proxy-related environment variables, the local address and the
+    # configured archive_host are excluded from http[s] proxying and general
+    # http[s] proxying it set according to the user-given --proxy option.
+    environment_setup="archive_host=\`awk '/^deb .*(debian|ubuntu)/ { split(\$2, u, \"/\"); print u[3]; exit }' \"\$root/etc/apt/sources.list\"\`; echo \"no_proxy=\\\"localhost,127.0.0.1,localaddress,.localdomain.com,\${archive_host}\\\"\nhttp_proxy=\\\"${PROXY}\\\"\nhttps_proxy=\\\"${PROXY}\\\"\nHTTP_PROXY=\\\"${PROXY}\\\"\nHTTPS_PROXY=\\\"${PROXY}\\\"\" >> /etc/environment"
+
     # generate cloud-init user data; mostly for manage_etc_hosts, but also get
     # rid of some unnecessary stuff in the VM
     #
@@ -256,6 +283,8 @@
  - apt-get clean
  # Do a dist-upgrade:
  - DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
+ # Set extra environment variables for testing.
+ - ${environment_setup}
 EOF
 
     EXTRA_OPTS=''
@@ -366,13 +395,13 @@
     fi
 
     DELETE_CMD="neutron security-group-delete $SRVNAME"
-    if ! type neutron >/dev/null || ! neutron security-group-list >/dev/null; then
+    if ! type neutron >/dev/null || ! neutron security-group-list >/dev/null 2>&1; then
         DELETE_CMD="nova secgroup-delete $SRVNAME"
     fi
 
     debug "Deleting $SRVNAME security-group"
     retry=3
-    while ! eval "$DELETE_CMD"; do
+    while ! eval "$DELETE_CMD" >/dev/null 2>&1; do
         retry=$(( retry - 1 ))
         if [ $retry -le 0 ]; then
             error "Timed out deleting secgroup. Aborting!"