Merge lp:~paulcz/charms/precise/logstash-indexer/trunk into lp:charms/logstash-indexer

[Paul Czarkowski]

* Updated to support logstash 1.2.x

[lp:~paulcz/charms/precise/logstash-indexer/trunk]

8. By Paul Czarkowski on 2013-10-08

amqp relationship

9. By Paul Czarkowski on 2013-10-16

filters

10. By Paul Czarkowski on 2013-10-18

remove dropping message

[Marco Ceppi (marcoceppi)]

LGTM +1

review: Approve

=== modified file 'README.md'
--- README.md	2012-11-18 22:27:48 +0000
+++ README.md	2013-10-18 15:38:52 +0000
@@ -19,37 +19,37 @@
 example 1 - Just the Indexer + Kibana
 =====================================
 
-juju deploy --repository=logstash local:precise/logstash-indexer
-juju deploy --repository=logstash local:precise/kibana
-juju add-relation kibana logstash-indexer:rest
-juju expose kibana
+    juju deploy --repository=logstash local:precise/logstash-indexer
+    juju deploy --repository=logstash local:precise/kibana
+    juju add-relation kibana logstash-indexer:rest
+    juju expose kibana
 
 http://ip-of-kibana
 
 example 2 - Indexer + 2 x ElasticSearch + Kibana
 ============================================
 
-juju deploy --repository=logstash local:precise/elasticsearch
-juju add-unit elasticsearch
-juju deploy --repository=logstash local:precise/logstash-indexer
-juju add-relation logstash-indexer elasticsearch:cluster
-juju deploy --repository=logstash local:precise/kibana
-juju add-relation kibana elasticsearch:rest
-juju expose kibana
+    juju deploy --repository=logstash local:precise/elasticsearch
+    juju add-unit elasticsearch
+    juju deploy --repository=logstash local:precise/logstash-indexer
+    juju add-relation logstash-indexer elasticsearch:cluster
+    juju deploy --repository=logstash local:precise/kibana
+    juju add-relation kibana elasticsearch:rest
+    juju expose kibana
 
 http://ip-of-kibana
 
 example 3  - Agent + Indexer + 2 x ElasticSearch + Kibana
 =============================================
 
-juju deploy --repository=logstash local:precise/elasticsearch
-juju add-unit elasticsearch
-juju deploy --repository=logstash local:precise/logstash-indexer
-juju add-relation logstash-indexer elasticsearch:cluster
-juju deploy --repository=logstash local:precise/kibana
-juju add-relation kibana elasticsearch:rest
-juju expose kibana
-juju deploy --repository=logstash local:precise/logstash-agent
-juju add-relation logstash-agent logstash-indexer:input
+    juju deploy --repository=logstash local:precise/elasticsearch
+    juju add-unit elasticsearch
+    juju deploy --repository=logstash local:precise/logstash-indexer
+    juju add-relation elasticsearch:cluster logstash-indexer
+    juju deploy --repository=logstash local:precise/kibana
+    juju add-relation elasticsearch:rest kibana
+    juju expose kibana
+    juju deploy --repository=logstash local:precise/logstash-agent
+    juju add-relation logstash-agent logstash-indexer:input
 
 http://ip-of-kibana
\ No newline at end of file

=== added file 'files/charm/50-filters.conf'
--- files/charm/50-filters.conf	1970-01-01 00:00:00 +0000
+++ files/charm/50-filters.conf	2013-10-18 15:38:52 +0000
@@ -0,0 +1,40 @@
+filter {
+  
+  if [type] == "sensu" {
+    date {
+      match => ["timestamp", "ISO8601"]
+    }
+    mutate {
+      remove_field => [ "timestamp" ]
+    }    
+  }
+
+  if [type] == "web-access" {
+    grok {
+      match => ["message", "%{COMBINEDAPACHELOG}"]
+    }
+    date {
+      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss" ]
+    }
+    geoip {
+      source => "clientip"
+    }    
+    mutate {
+      remove_field => [ "timestamp" ]
+    }
+  }
+
+  if [type] == "syslog" {
+    grok {
+      overwrite => "message"
+      match => {
+        "message" => "%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:host} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?%{GREEDYDATA:message}"
+      }
+    }
+    syslog_pri { }
+    date {
+      match => [ "timestamp",  "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
+    }
+  }  
+
+}
\ No newline at end of file

=== modified file 'files/charm/logstash-indexer.conf'
--- files/charm/logstash-indexer.conf	2012-11-18 22:27:48 +0000
+++ files/charm/logstash-indexer.conf	2013-10-18 15:38:52 +0000
@@ -18,5 +18,5 @@
 console log
 
 script
-	java -jar BASEPATH/bin/logstash-monolithic.jar agent -f BASEPATH/conf.d/ --log BASEPATH/logstash-indexer.log -v
+	java -jar BASEPATH/bin/logstash.jar agent -f BASEPATH/conf.d/ --log BASEPATH/logstash-indexer.log
 end script
\ No newline at end of file

=== added file 'hooks/amqp-relation-changed'
--- hooks/amqp-relation-changed	1970-01-01 00:00:00 +0000
+++ hooks/amqp-relation-changed	2013-10-18 15:38:52 +0000
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -eux # -x for verbose logging to juju debug-log
+HOME=`dirname $0`
+. $HOME/common.sh
+
+juju-log "logstash_agent - Relationship Advice -> $JUJU_REMOTE_UNIT"
+juju-log Relation settings:
+relation-get
+juju-log Relation members:
+relation-list
+
+RABBIT_HOST=$(relation-get private-address)
+RABBIT_PASS=$(relation-get password)
+
+if [[ -z $RABBIT_HOST ]] || [[ -z $RABBIT_PASS ]] ; then
+  juju-log "logstash_agent - amqp_changed: rabbit_host||rabbit_password not set."
+  exit 0
+fi
+
+
+clustered=$(relation-get clustered)
+if [[ -n "$clustered" ]] ; then
+  juju-log "logstash_agent - ampq_changed: Configuring for access to haclustered rabbitmq service."
+  RABBIT_VIP=$(relation-get vip)
+  [[ -z "$vip" ]] && juju-log "logstash_agent - amqp_changed: Clustered but no vip." && exit 0
+  RABBIT_HOST="$RABBIT_VIP"
+fi
+
+
+cat > ${BASEPATH}/conf.d/input_rabbit.conf << EOF
+input {
+  rabbitmq {
+    exchange      => "logstash"
+    host          => "${RABBIT_HOST}"
+    user          => "${RABBIT_USER}"
+    password      => "${RABBIT_PASS}"
+    vhost         => "${RABBIT_VHOST}"
+  }
+}
+EOF
+
+[ "$(initctl status logstash-indexer | grep running)" ] && initctl restart logstash-indexer || initctl start logstash-indexer

=== added file 'hooks/amqp-relation-joined'
--- hooks/amqp-relation-joined	1970-01-01 00:00:00 +0000
+++ hooks/amqp-relation-joined	2013-10-18 15:38:52 +0000
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -eux # -x for verbose logging to juju debug-log
+HOME=`dirname $0`
+. $HOME/common.sh
+
+relation-set username=${RABBIT_USER}
+relation-set vhost=${RABBIT_VHOST}
+
+juju-log "logstash-indexer - amqp_joined: requesting credentials for $RABBIT_USER"
\ No newline at end of file

=== modified file 'hooks/common.sh'
--- hooks/common.sh	2012-11-18 22:27:48 +0000
+++ hooks/common.sh	2013-10-18 15:38:52 +0000
@@ -3,13 +3,15 @@
 #set -eux # -x for verbose logging to juju debug-log
 
 # Set CONFIG constants
-VERSION="1.1.4"
-BASEURL="https://logstash.objects.dreamhost.com/release"
-CHECKSUM="68ac6c953aad026752f69ae7c15b13f4"
+VERSION="1.2.1"
+BASEURL="https://download.elasticsearch.org/logstash/logstash"  
+CHECKSUM="863272192b52bccf1fc2cf839a888eaf"
 BASEPATH="/opt/logstash"
 USER="logstash"
 HOST=`unit-get private-address`
 CLUSTER_NAME="elasticsearch"
+RABBIT_USER="logstash"
+RABBIT_VHOST="logstash"
 
 # Grab CONFIG variables
 # Example :

=== modified file 'hooks/install'
--- hooks/install	2012-11-18 22:27:48 +0000
+++ hooks/install	2013-10-18 15:38:52 +0000
@@ -7,7 +7,7 @@
 juju-log "Installing required Packages via apt-get - java"
 # was giving errors trying to install jre without update first.
 apt-get update
-apt-get -y install default-jre-headless curl redis-server
+apt-get -y install openjdk-7-jre-headless curl redis-server
 # set redis listener
 sed -i.bak -e "s/bind 127.0.0.1/#bind ${HOST}/" /etc/redis/redis.conf
 juju-log "Install upstart init script"
@@ -21,14 +21,14 @@
 mkdir -p ${BASEPATH}/data
 juju-log "install logstash and create config files"
 # Check if a local copy of the jar exists
-if [ -f ${HOME}/../files/charm/logstash-${VERSION}-monolithic.jar ]
+if [ -f ${HOME}/../files/charm/logstash-${VERSION}-flatjar.jar ]   logstash-1.2.1-flatjar.jar
 then
-  install -o root -g root -m 0644 ${HOME}/../files/charm/logstash-${VERSION}-monolithic.jar ${BASEPATH}/bin/logstash-monolithic.jar
+  install -o root -g root -m 0644 ${HOME}/../files/charm/logstash-${VERSION}.jar ${BASEPATH}/bin/logstash.jar
 else
   # Download logstash monolithic jar and checksum
-  wget -q -O ${BASEPATH}/bin/logstash-monolithic.jar ${BASEURL}/logstash-${VERSION}-monolithic.jar
-  if [[ $(md5sum "${BASEPATH}/bin/logstash-monolithic.jar" | cut -d " " -f 1) != ${CHECKSUM} ]]; then
-    juju-log --log-level CRITICAL "Checksum of ${BASEPATH}/bin/logstash-monolithic.jar failed"
+  wget -q -O ${BASEPATH}/bin/logstash.jar ${BASEURL}/logstash-${VERSION}-flatjar.jar
+  if [[ $(md5sum "${BASEPATH}/bin/logstash.jar" | cut -d " " -f 1) != ${CHECKSUM} ]]; then
+    juju-log --log-level CRITICAL "Checksum of ${BASEPATH}/bin/logstash.jar failed"
     exit 1
   fi
 fi
@@ -36,6 +36,8 @@
 echo 'input { redis { host => "127.0.0.1" data_type => "list" type => "redis" tags => ["redis"] key => "logstash" message_format => "json_event" } } ' > ${BASEPATH}/conf.d/input-redis.conf
 echo 'output {  elasticsearch {    embedded => true  } }' > ${BASEPATH}/conf.d/output-elasticsearch.conf
 
+install -o root -g root -m 0644 ${HOME}/../files/charm/50-filters.conf ${BASEPATH}/conf.d/50-filters.conf
+
 juju-log "create log file and pwn it"
 touch /var/log/logstash-indexer.log
 chown ${USER}. /var/log/logstash-indexer.log

=== modified file 'metadata.yaml'
--- metadata.yaml	2012-11-18 22:27:48 +0000
+++ metadata.yaml	2013-10-18 15:38:52 +0000
@@ -4,10 +4,14 @@
 description: |
   Installs the logstash indexer.   Initially stand alone version,  
   but would like to add charms and hooks for ElasticSearch, 
-  Kibana, and other stuffs.  
+  Kibana, and other stuffs.
+categories:
+  - applications
 requires:
   cluster:
     interface: elasticsearch
+  amqp:
+    interface: rabbitmq    
 provides:
   input:
     interface: redis

=== modified file 'revision'
--- revision	2012-11-01 04:07:22 +0000
+++ revision	2013-10-18 15:38:52 +0000
@@ -1,1 +1,1 @@
-0
+1