Merge lp:~jtv/orchestra/odev-executable into lp:~orchestra/orchestra/odev

[Jeroen T. Vermeulen]

Executable HOWTO; fix architecture guess in setup.py; less wget output.

Here's a bundle of changes I made locally:

 * Fix a stupid mistake I made in the architecture-guessing code in setup.py. It broke.

 * Reduce wget's progress output. It's nice to have some, but the volume was insane.

 * Turn HOWTO into a proper script. The difference is small, the convenience great.

 * Wait a bit between firing up Cobbler and populating its database.

 * Clean up bits of code. Easier to follow, I hope.

This gives me an odev setup that I can get running just by executing HOWTO. There's been just one wart so far: I had "sleep 10" in the HOWTO before, and I guess that wasn't enough. The setup.py run to populate the database would fail with "no route to host," but afterwards I could ssh in and also access the web UI. I guess it just needs a little longer; 20 seconds is a wild stab in the dark.

Jeroen

=== modified file 'HOWTO' (properties changed: -x to +x)
--- HOWTO	2012-01-25 14:54:08 +0000
+++ HOWTO	2012-01-27 19:52:03 +0000
@@ -1,46 +1,57 @@
+#! /bin/bash -e
+#
+# This file documents how to get odev running on your system.  But it's also
+# a script; you may find that you can just run it and get a working setup.
+#
 ## install some dependencies
-$ pkgs=""
-$ pkgs="$pkgs genisoimage coreutils" # for cloud-init's 'make-iso'
-$ pkgs="$pkgs python-libvirt libvirt-bin" # for libvirt interaction
-$ pkgs="$pkgs socat" # for libvirt-> cobbler
-$ pkgs="$pkgs python-cheetah" # for setup.py
+pkgs=""
+pkgs="$pkgs genisoimage coreutils" # for cloud-init's 'make-iso'
+pkgs="$pkgs python-libvirt libvirt-bin" # for libvirt interaction
+pkgs="$pkgs socat" # for libvirt-> cobbler
+pkgs="$pkgs python-cheetah" # for setup.py
 
-$ sudo apt-get update
-$ sudo apt-get install -y $pkgs </dev/null
+sudo apt-get update -qq || /bin/true
+sudo apt-get install -y $pkgs </dev/null
 
 ## add your user to libvirtd group
-$ sudo adduser $USER libvirtd
-$ sudo adduser $USER kvm # this is really only necessary for zimmer-build
+sudo adduser $USER libvirtd
+sudo adduser $USER kvm # this is really only necessary for zimmer-build
 
 ##
 ## NOTE: you have to log out and log back in for this to take affect
 ##
 
 ## build a zimmer image by following readme in zimmer-build
-$ cd zimmer-build
-$ ./build zimmer-disk0.img
-$ cd ..
+cd zimmer-build
+./build zimmer-disk0.img
+cd ..
 
 ## create libvirt xml files for nodes, zimmer, network
-$ ./setup.py libvirt-setup
+./setup.py libvirt-setup
 
 ## start odev-net network
-$ virsh -c qemu:///system net-start odev-net
+virsh -c qemu:///system net-start odev-net
 
 ## create zimmer disk image qcow backing against pristine version
-$ qemu-img create -f qcow2 -b zimmer-build/zimmer-disk0.img zimmer-disk0.img
+qemu-img create -f qcow2 -b zimmer-build/zimmer-disk0.img zimmer-disk0.img
 
 ## start zimmer instance / orchestra server
-$ virsh -c qemu:///system start zimmer
-
-## Now, 
-## * you can ssh ubuntu@192.168.123.2 (password passw0rd)
-##   Then run 'ssh-import-id' or sudo.
-## * http://192.168.123.2/cobbler_web should have cobbler web ui,
-##   log in with 'cobbler:xcobbler'
+virsh -c qemu:///system start zimmer
+
+echo
+echo "Starting orchestra server."
+echo "You can now ssh ubuntu@192.168.123.2 (password: passw0rd)."
+echo "If you do that, you may run 'ssh-import-id' to import your ssh key."
+echo
+echo "Access the cobbler UI on http://192.168.123.2/cobbler_web"
+echo "and log in with 'cobbler:xcobbler'."
+
+## Give the server some time to start up.
+sleep 20
+echo
 
 ## populate the nodes into the cobbler server
-$ ./setup.py cobbler-setup
+./setup.py cobbler-setup
 
 ## * libvirt from the cobbler system:
 ##   after 'cobbler-setup' above is done, the cobbler system will know about
@@ -52,7 +63,7 @@
 ##   to forward tcp connections on 192.168.123.1:65001 to the libvirt unix
 ##   socket .  It restricts connections to zimmer's IP address.
 
-$ socat -d -d \
+socat -d -d \
      TCP4-LISTEN:65001,bind=192.168.123.1,range=192.168.123.2/32,fork \
      UNIX-CONNECT:/var/run/libvirt/libvirt-sock
 

=== modified file 'setup.py'
--- setup.py	2012-01-26 16:41:33 +0000
+++ setup.py	2012-01-27 19:52:03 +0000
@@ -1,12 +1,9 @@
 #!/usr/bin/python
 
 import yaml
-import Cheetah
 import os
 import re
 import sys
-import copy
-import pprint
 import libvirt
 from Cheetah.Template import Template
 import subprocess
@@ -49,7 +46,8 @@
 		return ret
 
 	def toLibVirtXml(self):
-		return(Template(file=self.template, searchList=[self.dictInfo()]).respond())
+		template = Template(file=self.template, searchList=[self.dictInfo()])
+		return template.respond()
 
 class Node(Domain):
 	def _setcfg(self, cfg, num):
@@ -71,7 +69,8 @@
 		self.mem = cfg['mem'] * 1024
 
 def renderSysDom(config, syscfg, stype="node"):
-	return(Template(file=syscfg['template'], searchList=[config, syscfg]).respond())
+	template = Template(file=syscfg['template'], searchList=[config, syscfg])
+	return template.respond()
 
 # cobbler:
 #  ip: 2 # ip address must be in dhcp range
@@ -172,8 +171,7 @@
 	"""Get the system architecture for use in the cobbler setup profile."""
 	# This should, for any given system, match what the zimmer-build
 	# script does to determine the right architecture.
-	arch_text = subprocess.check_output(
-        ['/bin/uname', '-m'], stdout=subprocess.PIPE)
+	arch_text = subprocess.check_output(['/bin/uname', '-m'])
 	if re.match('i.86', arch_text):
 		return 'i386'
 	else:
@@ -186,12 +184,12 @@
 	profile = "precise-%s-juju" % arch
 	
 	cob = System(config, "zimmer")
-	server = xmlrpclib.Server("http://%s/cobbler_api" % cob.ipaddr)
+	cobbler_url = "http://%s/cobbler_api" % cob.ipaddr
+	print("Connecting to %s." % cobbler_url)
+	server = xmlrpclib.Server(cobbler_url)
 	token = server.login("cobbler","xcobbler")
 
-	systems = [ ]
-	for node in NODES_RANGE:
-		systems.append(Node(config, node))
+	systems = [Node(config, node) for node in NODES_RANGE]
 
 	for system in systems:
 		cobbler_addsystem(server, token, system, profile, hostip)
@@ -201,17 +199,17 @@
 	cfg_file = "settings.cfg"
 
 	if len(sys.argv) == 1:
-		print "Usage: setup.py action\n  action one of: libvirt-setup, cobbler-setup"
+		print(
+			"Usage: setup.py action\n"
+			"action one of: libvirt-setup, cobbler-setup")
 		sys.exit(1)
 
 	config = yaml_loadf(cfg_file)
 
 	if sys.argv[1] == "libvirt-setup":
 		libvirt_setup(config)
-		sys.exit(0)
 	elif sys.argv[1] == "cobbler-setup":
 		cobbler_setup(config)
-		sys.exit(0)
 
 if __name__ == '__main__':
 	main()

=== modified file 'zimmer-build/build'
--- zimmer-build/build	2012-01-26 17:08:39 +0000
+++ zimmer-build/build	2012-01-27 19:52:03 +0000
@@ -74,6 +74,34 @@
 	error "${@}"
 }
 
+# Strip directory path or URL from image name, and ensure a suffix of ".zimg".
+# So /tmp/foo.img, http://foo.bar/downloads/foo.zimg, and foo all become
+# "foo.zimg".
+base_zimg_name() {
+    local BASE_NAME
+    BASE_NAME="${1##*/}"
+    BASE_NAME="${BASE_NAME%.img}"
+    echo "${BASE_NAME%.zimg}.zimg"
+}
+
+# Download image file.
+# Parameters: source URL, filename to save to, directory to save to.
+download_img() {
+    local SOURCE DESTINATION
+    SOURCE="$1"
+    DESTINATION="$2"
+
+    if test -f "$DESTINATION"
+    then
+        fail "please delete $DESTINATION first or use --zimg|--img"
+    fi
+
+    debug 0 "downloading $SOURCE to $DESTINATION"
+    wget --progress=dot:mega "$SOURCE" -O "$DESTINATION.partial" &&
+        mv -- "$DESTINATION.partial" "$DESTINATION" ||
+        fail "failed to get $SOURCE"
+}
+
 short_opts="ho:v"
 long_opts="help,ci-bzr:,img:,log:,ud-file:,verbose,zimg:"
 getopt_out=$(getopt --name "${0##*/}" \
@@ -123,44 +151,33 @@
 	fail "failed to make tempdir"
 trap cleanup EXIT
 
-[ -d "$save_d" ] || mkdir -p "$save_d" || fail "failed to mkdir $save_d"
+mkdir -p "$save_d" || fail "failed to mkdir $save_d"
 
 if [ -n "$img" ]; then
 	# if img was given, then we assume good, its the backing image
 	[ -f "$img" ] || fail "$img (--img) is not a file"
 	debug 0 "using $img as uncompressed image"
 else
-	if [ -n "$zimg" ]; then
-		case "$zimg" in
-			http://*|https://*)
-				o_zimg="${zimg}"
-				zimg="${save_d}/${o_zimg##*/}"
-				zimg=${zimg%.img}; zimg=${zimg%.zimg}.zimg
-				[ -f "$zimg" ] &&
-					fail "please delete $zimg first or use --zimg|--img"
-				debug 0 "downloading $o_zimg to $zimg"
-				wget "$o_zimg" -O "$zimg.partial" &&
-					mv "$zimg.partial" "$zimg" || fail "failed to get $o_zimg"
-				;;
-			file://*)
-				o_zimg=${zimg}
-				zimg=${zimg#file://}
-				debug 0 "using file $o_zimg as zimg"
-				[ -f "$zimg" ] || fail "$zimg is not a file"
-				;;
-			*)  [ -f "$zimg" ] || fail "$zimg is not a file"
-				debug 0 "using file $o_zimg as zimg"
-				;;
-		esac
-	else
-		zimg="${save_d}/${DEF_ZIMG##*/}"
-		zimg=${zimg%.img}; zimg=${zimg%.zimg}.zimg
-		[ -f "$zimg" ] &&
-			fail "please delete $zimg first or use --zimg|--img"
-		debug 0 "downloading $DEF_ZIMG to $zimg"
-		wget "$DEF_ZIMG" -O "$zimg.partial" &&
-			mv "$zimg.partial" "$zimg" || fail "failed to get $DEF_ZIMG"
-	fi
+    if [ -z "$zimg" ]
+    then
+        zimg="$DEF_ZIMG"
+    fi
+	case "$zimg" in
+		http://*|https://*)
+			o_zimg="${zimg}"
+			zimg="${save_d}/`base_zimg_name ${zimg}`"
+			download_img "$o_zimg" "$zimg"
+			;;
+		file://*)
+			o_zimg=${zimg}
+			zimg=${zimg#file://}
+			debug 0 "using file $o_zimg as zimg"
+			[ -f "$zimg" ] || fail "$zimg is not a file"
+			;;
+		*)  [ -f "$zimg" ] || fail "$zimg is not a file"
+			debug 0 "using file $o_zimg as zimg"
+			;;
+	esac
 	img=${zimg%.zimg}.img
 	debug 0 "creating uncompressed img $img from $zimg"
 	qemu-img convert -O qcow2 "$zimg" "$img"