Merge lp:~smoser/charms/precise/virtual-maas/noju into lp:~virtual-maasers/charms/precise/virtual-maas/trunk

[Scott Moser]

remove / abstract the 'juju-ness'

This takes out the juju specific stuff, and replaces it with some wrappers.
The idea is that a user can conceivably do:
 * install server / launch instance
 * bzr branch my-branch
 * cd my-branch
 * sudo ./hooks/install
 * sudo ./hooks/config-changed
 * sudo ./hooks/start

It also adds some additional function such as:
 * configuration of user (rather than hard code 'ubuntu')
 * configuration of ephemeral images stream

[lp:~smoser/charms/precise/virtual-maas/noju]

71. By Scott Moser on 2013-04-30

xmaas-addssh-key: handle 'already added'

72. By Scott Moser on 2013-04-30

add example vmaas.conf

73. By Scott Moser on 2013-04-30

fix key added already

74. By Scott Moser on 2013-05-03

raring does not have /etc/maas/commissioning-user-data. mention that

=== modified file 'config.yaml'
--- config.yaml	2013-01-30 09:26:46 +0000
+++ config.yaml	2013-05-03 14:06:26 +0000
@@ -20,6 +20,11 @@
         description: |
             Space seperated list of releases to import
             MAAS pxe files for.
+    ephemeral_stream:
+        type: string
+        default: "released"
+        description: |
+            The stream to import ephemerals from ("daily" or "released")
     block-devices:
         type: string
         description: |
@@ -38,3 +43,8 @@
         default: em1
         description: |
             external interface to use for outbound routing
+    user:
+        type: string
+        default: ubuntu
+        description: |
+            the unprividged user to set up

=== modified file 'hooks/config-changed'
--- hooks/config-changed	2013-02-19 09:19:20 +0000
+++ hooks/config-changed	2013-05-03 14:06:26 +0000
@@ -1,6 +1,9 @@
 #!/bin/bash -e
 
-./scripts/setup-keys
-./scripts/setup-maas
-./scripts/setup-storage
-./scripts/setup-deployer
+. "${0%/*}/../scripts/_util"
+
+setup-keys
+setup-user
+setup-maas
+setup-storage
+setup-deployer

=== modified file 'hooks/install'
--- hooks/install	2013-02-19 08:55:47 +0000
+++ hooks/install	2013-05-03 14:06:26 +0000
@@ -1,26 +1,33 @@
 #!/bin/bash -e
 
+. "${0%/*}/../scripts/_util"
 
-source=$(config-get source)
+source=$(config_get source)
 
 if [ -n "$source" ]; then
-    add-apt-repository $source
+    add-apt-repository -y "$source"
 fi
 
-# TODO: Hacked version of isc-dhcp to ensure that
-# mac addresses always get assigned the sam IP
-# LP: 1069570
-add-apt-repository ppa:virtual-maasers/maas-updated-packages
-apt-get update
-
-apt-get install -y openvswitch-datapath-dkms
-apt-get install -y maas maas-region-controller maas-cluster-controller \
-                   maas-dhcp maas-dns maas-cli libvirt-bin kvm \
-                   gdisk lvm2 \
-                   python-keystoneclient python-novaclient \
-                   python-glanceclient python-quantumclient \
-                   python-cinderclient python-swiftclient \
-                   openvswitch-switch
+# maas-updated-packages ppa contains isc-dhcp for precise and quantal
+# that has the 'nouid' patch applied (LP: #1069570).
+# It may contain other packages also.
+add-apt-repository -y ppa:virtual-maasers/maas-updated-packages
+apt-get update -q
+
+maas_pkgs=(
+  maas maas-region-controller maas-cluster-controller
+  maas-dhcp maas-dns maas-cli
+)
+
+pkgs=(
+   openvswitch-datapath-dkms openvswitch-switch
+   libvirt-bin kvm
+   gdisk lvm2
+   linux-image-extra-virtual # if a cloud-guest, get kvm module
+)
+
+DEBIAN_FRONTEND=noninteractive \
+   apt-get install --quiet --assume-yes "${maas_pkgs[@]}" "${pkgs[@]}" 
 
 # Fixup nested virtualization and permissions on /dev/kvm
 # LP: 1092715

=== modified file 'hooks/start'
--- hooks/start	2013-01-23 18:42:03 +0000
+++ hooks/start	2013-05-03 14:06:26 +0000
@@ -1,6 +1,8 @@
 #!/bin/bash
 
-[ -b /dev/virtualmaas/instances ] && {
+. "${0%/*}/../scripts/_util"
+
+if [ -b /dev/virtualmaas/instances ]; then
     mkdir -p /var/lib/virtual-maas
     mount /dev/virtualmaas/instances /var/lib/virtual-maas || true
-}
+fi

=== modified file 'hooks/upgrade-charm'
--- hooks/upgrade-charm	2013-01-23 14:57:37 +0000
+++ hooks/upgrade-charm	2013-05-03 14:06:26 +0000
@@ -1,4 +1,6 @@
 #!/bin/bash -e
 
-./hooks/install
-./hooks/config-changed
+. "${0%/*}/../scripts/_util"
+
+"${HOOKS_D}/install"
+"${HOOKS_D}/config-changed"

=== added file 'scripts/_util'
--- scripts/_util	1970-01-01 00:00:00 +0000
+++ scripts/_util	2013-05-03 14:06:26 +0000
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+my_d=$(readlink -f "${BASH_SOURCE[0]%/*}")
+TOP_D="${my_d%/*}"
+SCRIPTS_D="${TOP_D}/scripts"
+HOOKS_D="${TOP_D}/hooks"
+tildame="~$(whoami)"
+HOME=$(sh -c "echo $tildame")
+
+add_pathd() {
+    local p=":$PATH:"
+    [ "${p#*:${1%/}:}" != "$p" ] || PATH="$1:$PATH"
+}
+
+as_unpriv_user() {
+    local user="${_VMAAS_UNPRIV_USER}"
+    if [ -z "$user"]; then
+        user=$(config_get "user")
+        [ -z "$user" ] && user="ubuntu"
+    fi
+    sudo -Hu "$user" "$@"
+}
+
+error() { echo "$@" 1>&2; }
+debug() {
+   local level="$1"
+   shift
+   error "$@"
+}
+
+if [ -z "$JUJU_UNIT_NAME" ]; then
+    if [ -z "$VMAAS_CONFIG" ]; then
+        for f in "${TOP_D}/vmaas.conf" "/etc/vmaas.conf"; do
+            [ -f "$f" ] && { export VMAAS_CONFIG="$f"; break; }
+        done
+    fi
+    if [ -z "$VMAAS_OPTIONS" ]; then
+        for f in "${TOP_D}/config.yaml" "/etc/vmaas-options.yaml"; do
+            [ -f "$f" ] && { export VMAAS_OPTIONS="$f"; break; }
+        done
+    fi
+fi
+
+add_pathd "$SCRIPTS_D"
+
+# vi: ts=4 expandtab syntax=sh

=== added file 'scripts/config_get'
--- scripts/config_get	1970-01-01 00:00:00 +0000
+++ scripts/config_get	2013-05-03 14:06:26 +0000
@@ -0,0 +1,51 @@
+#!/usr/bin/python
+##
+## Indirection layer around juju's "config-get".
+##
+## supports "noju" and just reading from a config file
+
+import sys, os
+
+def noju_config_get(name):
+    import yaml
+    ## Read an option out of a juju config formated file
+    ##  config file is specified in VMAAS_CONFIG environment variable
+    ##  if VMAAS_OPTIONS is set in environment, read defaults from it
+    cfg_file = os.environ.get("VMAAS_CONFIG")
+    
+    if not cfg_file:
+        dirs = ["/etc"]
+        for d in dirs:
+            fp = os.path.join(d, 'vmaas.conf')
+            if os.path.exists(fp):
+                cfg_file = fp
+                break
+
+    if cfg_file and os.path.exists(cfg_file):
+        cfg = yaml.safe_load(open(cfg_file).read())
+        # assume juju_unit_name would be 'vmaas'
+        if 'vmaas' in cfg:
+            cfg = cfg['vmaas']
+        elif len(cfg) == 1:
+            # if there was just one top level entry, take it
+            cfg = cfg[cfg.keys()[0]]
+    else:
+        cfg = {}
+
+    options_file = os.environ.get("VMAAS_OPTIONS", None)
+    if options_file:
+        options = yaml.safe_load(open(options_file).read())["options"]
+    else:
+        options = {}
+
+    print cfg.get(name, options.get(name, {}).get('default', ""))
+
+def juju_config_get(name):
+    os.execvp('config-get', ('config-get', name))
+
+if os.environ.get("JUJU_UNIT_NAME"):
+    config_get = juju_config_get
+else:
+    config_get = noju_config_get
+
+config_get(sys.argv[1])

=== modified file 'scripts/setup-deployer'
--- scripts/setup-deployer	2013-01-24 18:43:41 +0000
+++ scripts/setup-deployer	2013-05-03 14:06:26 +0000
@@ -1,16 +1,5 @@
 #!/bin/bash -e
 
-su -c "mkdir -p /home/ubuntu/tools" ubuntu
-
-[ -d /home/ubuntu/tools/juju-deployer ] || {
-    su -c "bzr branch lp:juju-deployer /home/ubuntu/tools/juju-deployer" ubuntu
-}
-
-[ -d /home/ubuntu/tools/maas-libvirt-utils ] || {
-    su -c "bzr branch lp:~virtual-maasers/+junk/maas-libvirt-utils /home/ubuntu/tools/maas-libvirt-utils" ubuntu
-}
-
-[ -d /home/ubuntu/tools/openstack-ubuntu-testing ] || {
-    su -c "bzr branch lp:openstack-ubuntu-testing /home/ubuntu/tools/openstack-ubuntu-testing" ubuntu
-}
-
+. "${0%/*}/../scripts/_util"
+
+:

=== modified file 'scripts/setup-keys'
--- scripts/setup-keys	2013-02-19 09:19:20 +0000
+++ scripts/setup-keys	2013-05-03 14:06:26 +0000
@@ -1,7 +1,8 @@
 #!/bin/bash -e
 
-keys=$(config-get ssh-keys)
+. "${0%/*}/../scripts/_util"
+
+keys=$(config_get ssh-keys)
 if [ -n "$keys" ]; then
-    su -c "ssh-import-id $keys" ubuntu
+    as_unpriv_user sh -c "ssh-import-id $keys"
 fi
-

=== modified file 'scripts/setup-maas'
--- scripts/setup-maas	2013-04-08 17:00:47 +0000
+++ scripts/setup-maas	2013-05-03 14:06:26 +0000
@@ -1,5 +1,7 @@
 #!/bin/bash -e
 
+. "${0%/*}/../scripts/_util"
+
 BRIDGE="maasbr0"
 
 MAAS_GW="192.168.64.1"
@@ -8,7 +10,8 @@
 MAAS_RANGE="192.168.64.100,192.168.74.254"
 MAAS_DHCP_INT="$BRIDGE"
 MAAS_NEXT_SERVER="$MAAS_GW"
-EXTIF=$(config-get ext-int)
+MAAS_USER="ubuntu"
+EXTIF=$(config_get ext-int)
 
 if sg libvirtd -c "virsh -c qemu:///system net-list" | grep -q $BRIDGE; then
     echo "Bridge already defined, destroying"
@@ -74,7 +77,9 @@
 ## maas-dhcp-server may run before libvirt-bin.
 ## if it does, then it will fail because the 'maasbr0' interface
 ## does not yet exist
-tee -a /etc/init/libvirt-maas-dhcpd-restart.conf <<"EOF"
+f="/etc/init/libvirt-maas-dhcpd-restart.conf"
+debug 1 "writing $f"
+cat > "$f" <<"EOF"
 description "restart dhcpd"
 start on started libvirt-bin
 task
@@ -84,7 +89,9 @@
 end script
 EOF
 
-tee /usr/local/bin/xmaas-get-user-creds <<"EOF"
+f="/usr/local/bin/xmaas-get-user-creds"
+debug 1 "writing $f"
+cat > "$f" <<"EOF"
 #!/usr/bin/python
 import sys, os
 sys.path.insert(0, "/usr/share/maas")
@@ -98,10 +105,12 @@
 token = admin.tokens.all()[0]
 print convert_tuple_to_string(get_creds_tuple(token))
 EOF
-sudo chmod 755 /usr/local/bin/xmaas-get-user-creds
-
-
-tee "/usr/local/bin/xmaas-addssh-key" <<"EOF"
+chmod 755 "$f"
+
+
+f=/usr/local/bin/xmaas-addssh-key
+debug 1 "writing $f"
+cat > "$f" <<"EOF"
 #!/usr/bin/python
 # this hackily adds an ssh key until LP: #1056860 is fixed
 import sys, os
@@ -113,20 +122,24 @@
 user = User.objects.get(username=sys.argv[1])
 kdata = open(sys.argv[2]).read()
 key = SSHKey(key=kdata, user=user)
-user.sshkey_set.add(key)
+try:
+    user.sshkey_set.add(key)
+except Exception as e:
+    if "already been added" in str(e.messages):
+        print "key from %s already added to user %s" % (sys.argv[2], user)
+        sys.exit(0)
+print "added key from %s to %s" % (sys.argv[2], user)
 user.save()
 EOF
-sudo chmod 755 /usr/local/bin/xmaas-addssh-key
-
-maas createadmin --username=ubuntu --password=ubuntu \
-   --email=[e-mail] 2>/dev/null || :
-
-CREDS=$(sudo xmaas-get-user-creds ubuntu)
+chmod 755 "$f"
+
+maas createadmin "--username=${MAAS_USER}" "--password=${MAAS_USER}" \
+   "--email=${MAAS_USER}@example.com" 2>/dev/null || :
+
+# Setup root and unprivledged user with 'maas-cli' usage
+CREDS=$(xmaas-get-user-creds "${MAAS_USER}")
 maas-cli login maaslocal http://localhost/MAAS/api/1.0 "$CREDS"
-
-# Setup ubuntu user
-cp /root/.maascli.db ~ubuntu/.maascli.db
-chown ubuntu:ubuntu ~ubuntu/.maascli.db
+as_unpriv_user maas-cli login maaslocal http://localhost/MAAS/api/1.0 "$CREDS"
 
 maas-cli maaslocal node-groups list > out.tmp
 sed -n '/^\[$/,$p' out.tmp > out
@@ -162,28 +175,36 @@
 service maas-dhcp-server restart
 
 
-releases=$(config-get releases)
-sed -i -e "s/^RELEASES.*/RELEASES=\"$releases\"/g" \
-       -e "s/.*ARCHES.*/ARCHES=\"amd64\/generic i386\/generic\"/g" \
-       -e "s/.*IMPORT_SQUASHFS.*/IMPORT_SQUASHFS=0/g" \
-          /etc/maas/import_pxe_files
-sed -i -e "s/.*RELEASES.*/RELEASES=\"precise\"/g" \
-          /etc/maas/import_ephemerals
+releases=$(config_get releases)
+stream=$(config_get ephemeral_stream)
+
+cfg="/etc/maas/import_pxe_files"
+[ -e "$cfg.dist" ] || cp "$cfg" "$cfg.dist"
+grep -q "${cfg}.vmaas" "$cfg" || echo ". ${cfg}.vmaas" >> "$cfg"
+
+cat > "${cfg}.vmaas" <<EOF
+RELEASES="$releases"
+ARCHES="amd64/generic i386/generic"
+IMPORT_SQUASHFS=0
+IMPORT_EPHEMERALS=1
+EOF
+
+cfg="/etc/maas/import_ephemerals"
+[ -e "$cfg.dist" ] || cp "$cfg" "$cfg.dist"
+grep -q "${cfg}.vmaas" "$cfg" || echo ". ${cfg}.vmaas" >> "$cfg"
+
+cat > "${cfg}.vmaas" <<EOF
+RELEASES="$releases"
+STREAM="$stream"
+EOF
+
 maas-import-pxe-files
 
-# add your ssh keys (LP: #1056860)
-[ -f ~ubuntu/.ssh/id_rsa.pub ] || {
-    su -c "ssh-keygen -N '' -f ~/.ssh/id_rsa" ubuntu
-    xmaas-addssh-key ubuntu ~ubuntu/.ssh/id_rsa.pub
-}
-
-# Turn off host key checking for managed KVM instances
-cat > ~ubuntu/.ssh/config << EOF
-Host *.virtual-maas.com
-    UserKnownHostsFile /dev/null
-    StrictHostKeyChecking no
-EOF
-chown ubuntu:ubuntu ~ubuntu/.ssh/config
+# add unpriv user's ssh keys
+# this has to be in subshell to get '~' expansion of variable
+as_unpriv_user sh -c 'cat ~/.ssh/id_rsa.pub' > pubkey.tmp
+xmaas-addssh-key ${MAAS_USER} pubkey.tmp
+rm -f pubkey.tmp
 
 # Setup system so MAAS can sudo virsh for virtual power control
 echo "maas ALL= NOPASSWD: /usr/bin/virsh" > /etc/sudoers.d/98-maas-virsh
@@ -194,8 +215,8 @@
 # Create juju environments file
 # Note the origin uses a pre-trunk serialized execution
 # branch to fix dpkg races between hooks in the same container
-mkdir -p ~ubuntu/.juju
-cat > ~ubuntu/.juju/environments.yaml << EOF
+as_unpriv_user sh -ec 'cd; mkdir -p .juju
+   cat > .juju/environments.yaml' << EOF
 default: precise
 environments:
   quantal:
@@ -220,7 +241,6 @@
     default-series: raring
     juju-origin: lp:juju
 EOF
-chown ubuntu:ubuntu ~ubuntu/.juju/environments.yaml
 
 ### fix enlistment script ipmi (LP: 1064527). do not attempt enlist
 sed -i.xdist \
@@ -229,9 +249,15 @@
    /usr/share/maas/preseeds/enlist_userdata
 
 ### fix commisssioing ipmi (LP: 1103716)
-sed -i.xdist -e 's,-z "power_settings",-z "$power_settings",' \
-    -e 's,\([^#]\)power_settings=$(,\1#power_settings=$(,' \
-    /etc/maas/commissioning-user-data
+if [ -e /etc/maas/commissioning-user-data ]; then
+    sed -i.xdist -e 's,-z "power_settings",-z "$power_settings",' \
+        -e 's,\([^#]\)power_settings=$(,\1#power_settings=$(,' \
+        /etc/maas/commissioning-user-data
+else
+    # FIXME: raring version of maas does not have this file
+    # likely/possibly have to make same changes to raring
+    :
+fi
 
 ### Max out the connections libvirt can deal with to avoid start/stop failures
 sed -i.xdist -E \
@@ -254,7 +280,7 @@
 options {
         directory "/var/cache/bind";
         forwarders {
-                $(config-get dns-server);
+                $(config_get dns-server);
         };
         auth-nxdomain no;    # conform to RFC1035
         listen-on-v6 { any; };
@@ -262,9 +288,6 @@
 EOF
 service bind9 restart
 
-# Add ubuntu to libvirtd group
-usermod -G libvirtd -a ubuntu || :
-
 # ensure access to the cloud-archive
 # LP: 1105192
 echo "ubuntu-cloud.archive.canonical.com" > /etc/squid-deb-proxy/mirror-dstdomain.acl.d/90-ca

=== modified file 'scripts/setup-storage'
--- scripts/setup-storage	2013-01-28 11:27:09 +0000
+++ scripts/setup-storage	2013-05-03 14:06:26 +0000
@@ -1,5 +1,6 @@
 #!/bin/bash -e
 
+. "${0%/*}/../scripts/_util"
 
 function check_mounted() {
     device=$1
@@ -12,7 +13,7 @@
 
 vgname=virtualmaas
 lvname=instances
-devices=$(config-get block-devices)
+devices=$(config_get block-devices)
 
 [ -z "$devices" ] && {
     echo "Devices not specificed"

=== added file 'scripts/setup-user'
--- scripts/setup-user	1970-01-01 00:00:00 +0000
+++ scripts/setup-user	2013-05-03 14:06:26 +0000
@@ -0,0 +1,37 @@
+#!/bin/bash -e
+
+. "${0%/*}/../scripts/_util"
+
+tool_info=(
+  "lp:juju-deployer|juju-deployer"
+  "lp:~virtual-maasers/+junk/maas-libvirt-utils|maas-libvirt-utils"
+  "lp:openstack-ubuntu-testing|openstack-ubuntu-testing"
+)
+
+as_unpriv_user sh -ec '
+   mkdir -p ~/tools
+   cd ~/tools
+   for tdata in "$@"; do
+      src=${tdata%|*}
+      dir=${tdata##*|}
+      [ -d "$dir" ] && { echo "$src already in $dir" 1>&2; continue; }
+      echo "branching $src" 1>&2
+      bzr branch --quiet "$src" "$dir"
+   done
+' -- "${tool_info[@]}"
+
+as_unpriv_user sh -c \
+    "[ -f ~/.ssh/id_rsa.pub ] || ssh-keygen -N '' -f ~/.ssh/id_rsa"
+
+user=$(config_get "user")
+adduser "$user" libvirtd
+adduser "$user" kvm
+
+# Turn off host key checking for managed KVM instances
+as_unpriv_user sh -c '
+    grep -q .virtual-maas.com ~/.ssh/config && exit 0
+    cat > ~/.ssh/config' <<EOF
+Host *.virtual-maas.com
+    UserKnownHostsFile /dev/null
+    StrictHostKeyChecking no
+EOF

=== added file 'vmaas.conf.example'
--- vmaas.conf.example	1970-01-01 00:00:00 +0000
+++ vmaas.conf.example	2013-05-03 14:06:26 +0000
@@ -0,0 +1,5 @@
+vmaas:
+    releases: raring
+    ephemeral_stream: "daily"
+    dns-server: 10.55.60.1 
+    ext-int: eth0