Merge lp:~ev/charms/precise/daisy-retracer/prodstack-prep into lp:~daisy-pluckers/charms/precise/daisy-retracer/trunk

[Evan]

=== modified file 'config.yaml'
--- config.yaml	2012-12-12 15:11:24 +0000
+++ config.yaml	2013-02-22 17:21:30 +0000
@@ -1,4 +1,28 @@
 options:
+  basedir:
+      default: "daisy.ubuntu.com"
+      type: string
+      description: Base directory for code
+  instance_type:
+      default: "production"
+      type: string
+      description: Instance type
+  user_code_runner:
+      default: "whoopsie"
+      type: string
+      description: The user that runs the code
+  group_code_runner:
+      default: "whoopsie"
+      type: string
+      description: The group that runs the code
+  user_code_owner:
+      default: "webops_deploy"
+      type: string
+      description: The user that owns the code
+  group_code_owner:
+      default: "webops_deploy"
+      type: string
+      description: The group that owns the code
   amqp-user:
       type: string
       default: 'daisy'
@@ -15,15 +39,54 @@
       type: string
       default: ''
       description: The AWS access key (EC2_ACCESS_KEY).
-  daisy-retracer-ec2-host:
+  daisy-retracer-s3-host:
       type: string
       default: ''
-      description: The EC2 IP address.
-  daisy-retracer-ec2-bucket:
+      description: The S3 IP address.
+  daisy-retracer-s3-bucket:
       type: string
       default: 'daisy_core_files'
-      description: The EC2 bucket.
+      description: The S3 bucket.
+  daisy-retracer-openstack-auth-url:
+      type: string
+      default: ''
+      description: |
+        The OpenStack authentication URL, e.g.:
+        https://keystone.yourdomain.com:443/v2.0/
+  daisy-retracer-openstack-username:
+      type: string
+      default: ''
+      description: The OpenStack username to authenticate with.
+  daisy-retracer-openstack-password:
+      type: string
+      default: ''
+      description: The OpenStack password to authenticate with.
+  daisy-retracer-openstack-tenant:
+      type: string
+      default: ''
+      description: |
+        The OpenStack tenant to use, e.g.: yourname_project.
+  daisy-retracer-openstack-region:
+      type: string
+      default: ''
+      description: The OpenStack region to use.
+  daisy-retracer-swift-bucket:
+      type: string
+      default: ''
+      description: |
+        The OpenStack Swift bucket to use, e.g.: daisy_core_files.
+        Setting this will use Swift instead of S3 for storing core files.
+  daisy-retracer-cassandra-host:
+      type: string
+      default: ''
+      description: |
+        Override relations to Cassandra with a specific host address. This
+        is a temporary measure until cross-environment relations are ready.
   daisy-retracer-architectures:
       type: string
       default: 'i386 amd64 armhf'
       description: Space separated list of architectures for which to process reports.
+  repository:
+      type: string
+      default: 'ppa:daisy-pluckers/prodstack-prep'
+      description: The location for the daisy-dependencies package.

=== added directory 'files'
=== added file 'hooks/amqp'
--- hooks/amqp	1970-01-01 00:00:00 +0000
+++ hooks/amqp	2013-02-22 17:21:30 +0000
@@ -0,0 +1,41 @@
+#!/bin/sh -eux
+
+. $(dirname $0)/common
+
+get_rabbit_settings () {
+    local rabbit_user="$(config-get amqp-user)"
+    local rabbit_vhost="$(config-get amqp-vhost)"
+    relation-set username="$rabbit_user"
+    relation-set vhost="$rabbit_vhost"
+}
+
+save_rabbit_settings () {
+    local rabbit_user="$(config-get amqp-user)"
+    local rabbit_vhost="$(config-get amqp-vhost)"
+    local rabbit_host="$(relation-get private-address)"
+    local rabbit_password="$(relation-get password)"
+    if [ -z "$rabbit_host" ] || [ -z "$rabbit_password" ]; then
+        juju-log "Missing AMQP host or password."
+        exit 0
+    fi
+    save_setting amqp_host "$rabbit_host"
+    save_setting amqp_username "$rabbit_user"
+    save_setting amqp_password "$rabbit_password"
+    save_setting amqp_vhost "$rabbit_vhost"
+
+    retracer_restart
+}
+
+COMMAND="$(basename $0)"
+
+case "$COMMAND" in
+    amqp-relation-joined)
+        get_rabbit_settings
+        ;;
+    amqp-relation-changed)
+        save_rabbit_settings
+        ;;
+    *)
+        juju-log "Command not recognized"
+        ;;
+esac

=== modified symlink 'hooks/amqp-relation-changed'
=== target changed u'relation-changed' => u'amqp'
=== modified symlink 'hooks/amqp-relation-joined'
=== target changed u'relation-changed' => u'amqp'
=== renamed file 'hooks/relation-changed' => 'hooks/cassandra'
--- hooks/relation-changed	2012-12-13 10:14:41 +0000
+++ hooks/cassandra	2013-02-22 17:21:30 +0000
@@ -2,61 +2,55 @@
 
 . $(dirname $0)/common
 
-cassandrahost="$(relation-get private-address)"
-nfshost="$(relation-get private-address)"
-rabbithost="$(relation-get private-address)"
-
-get_rabbit_settings () {
-    local rabbit_user="$(config-get amqp-user)"
-    local rabbit_vhost="$(config-get amqp-vhost)"
-    relation-set username="$rabbit_user"
-    relation-set vhost="$rabbit_vhost"
-}
-
-save_rabbit_settings () {
-    local rabbit_user="$(config-get amqp-user)"
-    local rabbit_vhost="$(config-get amqp-vhost)"
-    local rabbit_host="$(relation-get private-address)"
-    local rabbit_password="$(relation-get password)"
-    if [ -z "$rabbit_host" ] || [ -z "$rabbit_password" ]; then
-        juju-log "Missing AMQP host or password."
-        exit 0
-    fi
-    save_setting amqp_host "$rabbit_host"
-    save_setting amqp_username "$rabbit_user"
-    save_setting amqp_password "$rabbit_password"
-    save_setting amqp_vhost "$rabbit_vhost"
-}
-
-save_cassandra_settings () {
-    if ! grep -qs 'cassandra_hosts' $config; then
-        echo "cassandra_hosts = ['$cassandrahost']" >> $config
-    else
-        sed -i "s,cassandra_hosts = .*,cassandra_hosts = ['$cassandrahost']," $config
-    fi
-}
-
+address="$(relation-get private-address)"
 COMMAND="$(basename $0)"
 
-case "$COMMAND" in
-    amqp-relation-joined)
-        get_rabbit_settings
-        ;;
-    amqp-relation-changed)
-        save_rabbit_settings
-        ;;
-    db-relation-changed)
-        save_cassandra_settings
-        ;;
-    *)
-        juju-log "Command not recognized"
-        ;;
-esac
-
-if [ -n "$cassandrahost" ] && [ -n "$rabbithost" ]; then
-    juju-log "Restarting all retracers"
-    for job in `(cd /etc/init; ls retracer*.conf)`; do 
-        stop ${job%.conf} || true
-        start ${job%.conf}
-    done
-fi
+if [ -n "$(config-get daisy-cassandra-host)" ]; then
+    juju-log "A Cassandra host has been manually set. Ignoring the relation."
+    exit 0
+fi
+
+if [ -z "$address" ] && [ $COMMAND != "db-relation-broken" ]; then
+    juju-log "No address provided."
+    exit 0
+fi
+
+if [ "$COMMAND" = "db-relation-departed" ]; then
+    # First remove the address that just departed the relation, then remove the
+    # trailing comma from the list, if one exists.
+    sed -i -e "s|\(cassandra_hosts = .*\)'$address'[, ]*|\1|" \
+           -e "s|, ]|]|" $LOCAL_CONFIG_PATH
+
+elif [ "$COMMAND" = "db-relation-joined" ]; then
+    p="$LOCAL_CONFIG_PATH"
+
+    if ! grep -qs 'cassandra_hosts =' $p; then
+        # We do not have a cassandra_hosts variable. Set one with a new list.
+        echo "cassandra_hosts = ['$address']" >> $p
+
+    elif grep -qs "cassandra_hosts =.*']" $p; then
+        # We already have at least one host. Put the new host on the end of the
+        # list.
+        sed -i "s|\(cassandra_hosts.*\)]|\1, '$address']|" $p
+
+    elif grep -qs "cassandra_hosts =.*\[[ *]*\]" $p; then
+        # We have no hosts. Replace the entire line with a new single-element
+        # list.
+        sed -i "s|cassandra_hosts =.*|cassandra_hosts = ['$address']|" $p
+    fi
+
+    if "$(dirname $0)/create-keyspace" "$address"; then
+        OOPS_KEYSPACE=crashdb OOPS_HOST="$address" python \
+            /usr/share/pyshared/oopsrepository/schema.py
+        PYTHONPATH=$LOCAL_CONFIG_LOCATION:$CODE_LOCATION python \
+            $CODE_LOCATION/daisy/schema.py
+    fi
+
+elif [ "$COMMAND" = "db-relation-broken" ]; then
+    sed -i "s|cassandra_hosts =.*|cassandra_hosts = []|" $LOCAL_CONFIG_PATH
+fi
+
+# Restart the retracer, since we've changed its configuration.
+retracer_restart
+
+exit 0

=== modified file 'hooks/common'
--- hooks/common	2012-10-10 09:38:57 +0000
+++ hooks/common	2013-02-22 17:21:30 +0000
@@ -1,13 +1,62 @@
 #!/bin/sh
 
-config=/var/retracer/daisy/local_config.py
+BASEDIR="$(config-get basedir)"
+INSTANCE_TYPE="$(config-get instance_type)"
+USER_CODE_OWNER=$(config-get user_code_owner)
+GROUP_CODE_OWNER=$(config-get group_code_owner)
+USER_CODE_RUNNER=$(config-get user_code_runner)
+GROUP_CODE_RUNNER=$(config-get group_code_runner)
+ARCHITECTURES="$(config-get daisy-retracer-architectures)"
+DEPLOY_ROOT="/srv/$BASEDIR/$INSTANCE_TYPE"
+CODE_LOCATION="$DEPLOY_ROOT/daisy"
+LOCAL_CONFIG_LOCATION="$DEPLOY_ROOT/local_config"
+LOCAL_CONFIG_PATH="$LOCAL_CONFIG_LOCATION/local_config.py"
+LOG_DIR="/srv/$BASEDIR/$INSTANCE_TYPE-logs"
+APPORT_LOCATION="$DEPLOY_ROOT/apport"
+CACHE_DIR="$DEPLOY_ROOT/cache"
 
 save_setting () {
     key="$1"
     value="$2"
-    if ! grep -qs "$key" $config; then
-        echo "$key = '$value'" >> $config
+    if ! grep -qs "$key" $LOCAL_CONFIG_PATH; then
+        echo "$key = '$value'" >> $LOCAL_CONFIG_PATH
     else
-        sed -i "s,$key = .*,$key = '$value'," $config
+        sed -i "s,$key = .*,$key = '$value'," $LOCAL_CONFIG_PATH
     fi
 }
+
+add_user () {
+    local user="$1"
+    local group="$2"
+    id $user 2>/dev/null || adduser --quiet --system --disabled-password --ingroup $group $user
+}
+
+add_group () {
+    local group="$1"
+    getent group | grep -qs "^$group:" || addgroup --quiet "$group"
+}
+
+retracer_restart () {
+    # stop all retracers
+    juju-log "Stopping all retracers"
+    for job in `(cd /etc/init; ls retracer*.conf)`; do
+        stop ${job%.conf} || true
+        rm /etc/init/$job
+    done
+
+    # create upstart jobs for configured architectures, and start them
+    for arch in $ARCHITECTURES; do
+        juju-log "Creating upstart job for $arch retracer"
+        cat > /etc/init/retracer-$arch.conf << EOF
+description "error tracker retracer for $arch"
+env PYTHONPATH=$LOCAL_CONFIG_LOCATION:$CODE_LOCATION:$APPORT_LOCATION:$DEPLOY_ROOT
+export PYTHONPATH
+env PATH=$APPORT_LOCATION/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+export PATH
+exec $CODE_LOCATION/daisy/retracer.py --config-dir $CODE_LOCATION/retracer/config --sandbox-dir $CACHE_DIR --output $LOG_DIR/retracer-$arch.log --architecture $arch
+EOF
+        juju-log "Starting $arch retracer"
+        start retracer-$arch
+
+    done
+}

=== modified file 'hooks/config-changed'
--- hooks/config-changed	2012-12-12 15:11:24 +0000
+++ hooks/config-changed	2013-02-22 17:21:30 +0000
@@ -1,20 +1,27 @@
-#!/bin/sh
-set -eu
+#!/bin/sh -eux
 
 . $(dirname $0)/common
 
-# stop all retracers
-juju-log "Stopping all retracers"
-for job in `(cd /etc/init; ls retracer*.conf)`; do 
-    stop ${job%.conf} || true
-    rm /etc/init/$job
-done
-
 juju-log "Updating configuration settings"
+
+swift_bucket="$(config-get daisy-retracer-swift-bucket)"
+os_auth_url="$(config-get daisy-retracer-openstack-auth-url)"
+os_username="$(config-get daisy-retracer-openstack-username)"
+os_password="$(config-get daisy-retracer-openstack-password)"
+os_tenant="$(config-get daisy-retracer-openstack-tenant)"
+os_region="$(config-get daisy-retracer-openstack-region)"
+[ -n "$swift_bucket" ] && save_setting swift_bucket "$swift_bucket"
+[ -n "$os_auth_url" ] && save_setting os_auth_url "$os_auth_url"
+[ -n "$os_username" ] && save_setting os_username "$os_username"
+[ -n "$os_password" ] && save_setting os_password "$os_password"
+[ -n "$os_tenant" ] && save_setting os_tenant_name "$os_tenant"
+[ -n "$os_region" ] && save_setting os_region_name "$os_region"
+
 aws_secret_key="$(config-get daisy-retracer-aws-secret-key)"
 aws_access_key="$(config-get daisy-retracer-aws-access-key)"
-ec2_host="$(config-get daisy-retracer-ec2-host)"
-ec2_bucket="$(config-get daisy-retracer-ec2-bucket)"
+ec2_host="$(config-get daisy-retracer-s3-host)"
+ec2_bucket="$(config-get daisy-retracer-s3-bucket)"
+cassandra_host="$(config-get daisy-retracer-cassandra-host)"
 architectures="$(config-get daisy-retracer-architectures)"
 
 [ -n "$aws_secret_key" ] && save_setting aws_secret_key "$aws_secret_key"
@@ -22,10 +29,11 @@
 [ -n "$ec2_host" ] && save_setting ec2_host "$ec2_host"
 [ -n "$ec2_bucket" ] && save_setting ec2_bucket "$ec2_bucket"
 
-# create upstart jobs for configured architectures, and start them
-for arch in $architectures; do
-    juju-log "Creating upstart job for $arch retracer"
-    sed "s/@ARCH@/$arch/g" hooks/retracer.conf > /etc/init/retracer-$arch.conf
-    juju-log "Starting $arch retracer"
-    start retracer-$arch
-done
+# If the user has set daisy-cassandra-host to null, set cassandra_hosts back to
+# an empty list.
+[ -n "$cassandra_host" ] && save_cassandra "$cassandra_host" #|| \
+    #sed -i "s|cassandra_hosts =.*|cassandra_hosts = []|" $LOCAL_CONFIG_PATH
+
+retracer_restart
+
+exit 0

=== added symlink 'hooks/db-relation-broken'
=== target is u'cassandra'
=== removed symlink 'hooks/db-relation-changed'
=== target was u'relation-changed'
=== added symlink 'hooks/db-relation-departed'
=== target is u'cassandra'
=== added symlink 'hooks/db-relation-joined'
=== target is u'cassandra'
=== modified file 'hooks/install'
--- hooks/install	2012-12-12 15:11:24 +0000
+++ hooks/install	2013-02-22 17:21:30 +0000
@@ -1,37 +1,56 @@
 #!/bin/sh
-set -e
-
-mkdir -p /var/retracer/cache
-
-# TODO: move remaining packages over to daisy-seeds PPA
-add-apt-repository ppa:ev/whoopsie-daisy
-add-apt-repository ppa:daisy-pluckers/daisy-seeds
+set -eux
+
+. hooks/common
+
+add-apt-repository "$(config-get repository)"
 apt-get update
-# debootstrap is for building the chroot
-apt-get install -y retracer-dependencies debootstrap
-
-# get latest release of Apport
-if [ -d /var/retracer/apport ]; then
-    cd /var/retracer/apport
-    bzr pull
-else
-    bzr branch lp:apport /var/retracer/apport
-    cd /var/retracer/apport
-fi
-LATEST_TAG=`bzr tags | sort -nrk 2 | head -n1 | awk '{print $1}'`
-bzr uncommit --force -r tag:$LATEST_TAG
-bzr revert --no-backup
-bzr clean-tree --ignored --unknown --force
-# set up the apt/dpkg Apport packaging backend; setup.py build would need more
-# dependencies
-rm -f apport/packaging_impl.py
-ln -s ../backends/packaging-apt-dpkg.py apport/packaging_impl.py
-cd -
-
-# Retracer config
-if [ -d /var/retracer/daisy ]; then
-    (cd /var/retracer/daisy; bzr pull)
-else
-    bzr branch lp:~ev/daisy/s3 /var/retracer/daisy
-    cp /var/retracer/daisy/configuration.py /var/retracer/daisy/local_config.py
-fi
+apt-get install -y retracer-dependencies
+
+# Create groups
+add_group "$GROUP_CODE_OWNER"
+add_group "$GROUP_CODE_RUNNER"
+
+# Create users
+add_user "$USER_CODE_OWNER" "$GROUP_CODE_OWNER"
+add_user "$USER_CODE_RUNNER" "$GROUP_CODE_RUNNER"
+
+# Create directories
+mkdir -p "$CODE_LOCATION" "$LOCAL_CONFIG_LOCATION" "$LOG_DIR" "$APPORT_LOCATION" "$CACHE_DIR"
+
+# Unpack tarred Daisy code
+charm_dir="$(pwd)"
+tarball="$charm_dir/files/daisy.tgz"
+
+if [ -f "$tarball" ]; then
+    tar -C "$CODE_LOCATION" -xvf "$tarball"
+fi
+
+if [ ! -e "$LOCAL_CONFIG_PATH" ]; then
+    cp "$CODE_LOCATION/daisy/configuration.py" "$LOCAL_CONFIG_PATH"
+fi
+
+# Unpack tarred Apport snapshot
+apport="$charm_dir/files/apport.tgz"
+
+if [ -f "$apport" ]; then
+    tar -C "$APPORT_LOCATION" -xvf "$apport"
+
+    # set up the apt/dpkg Apport packaging backend; setup.py build would need more
+    # dependencies
+    cd "$APPORT_LOCATION"
+    rm -f apport/packaging_impl.py
+    ln -s ../backends/packaging-apt-dpkg.py apport/packaging_impl.py
+    cd -
+fi
+
+# Reset the Cassandra hosts setting. This may have been established by a
+# previous charm, but its contents are now invalid until the config-changed or
+# db-relation-* hooks fires. Its default value of 127.0.0.1 would persist
+# across db-relation-joined and db-relation-departed calls if left set.
+sed -i "s|cassandra_hosts =.*|cassandra_hosts = []|" "$LOCAL_CONFIG_PATH"
+
+chown -R $USER_CODE_OWNER:$USER_CODE_RUNNER "/srv/$BASEDIR"
+chown -R $USER_CODE_RUNNER:$GROUP_CODE_RUNNER "$CACHE_DIR"
+
+exit 0

=== removed symlink 'hooks/nfs-relation-changed'
=== target was u'relation-changed'
=== removed file 'hooks/retracer.conf'
--- hooks/retracer.conf	2012-12-12 15:11:24 +0000
+++ hooks/retracer.conf	1970-01-01 00:00:00 +0000
@@ -1,9 +0,0 @@
-description	"error tracker retracer for @ARCH@"
-
-# Use upstart because James prefers it over supervisord. jjo had a reason for
-# using supervisord - touch base.
-env PYTHONPATH=/var/retracer/daisy:/var/retracer/apport:/var/retracer
-export PYTHONPATH
-env PATH=/var/retracer/apport/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-export PATH
-exec /var/retracer/daisy/process_core.py --config-dir /var/retracer/daisy/retracer/config --sandbox-dir /var/retracer/cache --output /var/log/retracer-@ARCH@.log --architecture @ARCH@

=== modified file 'revision'
--- revision	2012-12-13 10:14:41 +0000
+++ revision	2013-02-22 17:21:30 +0000
@@ -1,1 +1,1 @@
-54
+59