Merge lp:~michael.nelson/charms/trusty/kibana/kibana4 into lp:~canonical-is-sa/charms/trusty/kibana/kibana4

[Michael Nelson]

Required changes for Kibana 4

This branch merges a community branch [1] (not landed upstream) to support kibana4, and then makes a few changes.

I'll comment inline below on the significant changes.

[1] https://code.launchpad.net/~chris.macnaughton/charms/trusty/kibana/version_bump/+merge/274029

[Michael Nelson (michael.nelson)]

Including inline comments.

[Tim Kuhlman (timkuhlman)]

lgtm, will merge.
Adding the comments was great, thanks.

review: Approve

=== modified file 'README.md'
--- README.md	2015-03-31 16:03:49 +0000
+++ README.md	2015-11-03 23:41:21 +0000
@@ -19,7 +19,7 @@
     juju deploy logstash-indexer
     juju add-relation logstash-indexer elasticsearch:cluster
     juju deploy kibana
-    juju add-relation kibana elasticsearch:rest
+    juju add-relation kibana elasticsearch
     juju expose kibana
 
 browse to http://ip-address to begin searching.
@@ -31,7 +31,7 @@
 configured values in 'kibana_source' and 'kibana_source_checksum', which
 default to::
 
-  https://download.elasticsearch.org/kibana/kibana/kibana-3.1.1.tar.gz
+  https://download.elastic.co/kibana/kibana/kibana-4.2.0-linux-x64.tar.gz
 
 It will, obviously, require internet access on deployed nodes.
 

=== modified file 'config.yaml'
--- config.yaml	2015-10-28 04:08:59 +0000
+++ config.yaml	2015-11-03 23:41:21 +0000
@@ -1,15 +1,11 @@
 options:
     kibana_source:
         type: string
-        default: 'https://download.elasticsearch.org/kibana/kibana/kibana-3.1.1.tar.gz'
+        default: 'https://download.elastic.co/kibana/kibana/kibana-4.2.0-linux-x64.tar.gz'
         description: URL to download the source for kibana
-    kibana_source_unpack_directory:
-        type: string
-        default: 'kibana-3.1.1'
-        description: the name of the directory the tar.gz will unpack to
     kibana_source_checksum:
         type: string
-        default: '74c6b8b79d1bfc7e4d5376e3c9ecb0fa1e79c986'
+        default: '094e197c687012211ae0e291faf3111d47e714d9'
         description: the sha1sum of the kibana_source
     port:
         type: int
@@ -20,7 +16,3 @@
         default: ''
         description: 'The url path of kibana. This is required if kibana is
         proxied at a non root path. e.g. http://example.com/kibana/'
-    elasticsearch_url:
-        type: string
-        default: '"http://"+window.location.hostname'
-        description: URL to for elasticsearch line in config.sh, defaults to '"http://"+window.location.hostname'

=== removed file 'files/charm/config.js'
--- files/charm/config.js	2015-03-26 03:45:25 +0000
+++ files/charm/config.js	1970-01-01 00:00:00 +0000
@@ -1,75 +0,0 @@
-/** @scratch /configuration/config.js/1
- *
- * == Configuration
- * config.js is where you will find the core Kibana configuration. This file contains parameter that
- * must be set before kibana is run for the first time.
- */
-define(['settings'],
-function (Settings) {
-  
-
-  /** @scratch /configuration/config.js/2
-   *
-   * === Parameters
-   */
-  return new Settings({
-
-    /** @scratch /configuration/config.js/5
-     *
-     * ==== elasticsearch
-     *
-     * The URL to your elasticsearch server. You almost certainly don't
-     * want +http://localhost:9200+ here. Even if Kibana and Elasticsearch are on
-     * the same host. By default this will attempt to reach ES at the same host you have
-     * kibana installed on. You probably want to set it to the FQDN of your
-     * elasticsearch host
-     */
-    elasticsearch: "http://"+window.location.hostname,
-
-    /** @scratch /configuration/config.js/5
-     *
-     * ==== default_route
-     *
-     * This is the default landing page when you don't specify a dashboard to load. You can specify
-     * files, scripts or saved dashboards here. For example, if you had saved a dashboard called
-     * `WebLogs' to elasticsearch you might use:
-     *
-     * default_route: '/dashboard/elasticsearch/WebLogs',
-     */
-    default_route     : '/dashboard/file/default.json',
-
-    /** @scratch /configuration/config.js/5
-     *
-     * ==== kibana-int
-     *
-     * The default ES index to use for storing Kibana specific object
-     * such as stored dashboards
-     */
-    kibana_index: "kibana-int",
-
-    /** @scratch /configuration/config.js/5
-     *
-     * ==== panel_name
-     *
-     * An array of panel modules available. Panels will only be loaded when they are defined in the
-     * dashboard, but this list is used in the "add panel" interface.
-     */
-    panel_names: [
-      'histogram',
-      'map',
-      'goal',
-      'table',
-      'filtering',
-      'timepicker',
-      'text',
-      'hits',
-      'column',
-      'trends',
-      'bettermap',
-      'query',
-      'terms',
-      'stats',
-      'sparklines'
-    ]
-  });
-});

=== added file 'files/charm/config.yml'
--- files/charm/config.yml	1970-01-01 00:00:00 +0000
+++ files/charm/config.yml	2015-11-03 23:41:21 +0000
@@ -0,0 +1,79 @@
+# Kibana is served by a back end server. This controls which port to use.
+port: 5601
+
+# The host to bind the server to.
+host: "0.0.0.0"
+
+# The Elasticsearch instance to use for all your queries.
+# This is proxied via /etc/nginx/sites-enabled/kibana_load_balancer
+elasticsearch_url: http://localhost:9200
+
+# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
+# then the host you use to connect to *this* Kibana instance will be sent.
+elasticsearch_preserve_host: true
+
+# Kibana uses an index in Elasticsearch to store saved searches, visualizations
+# and dashboards. It will create a new index if it doesn't already exist.
+kibana_index: ".kibana"
+
+# If your Elasticsearch is protected with basic auth, this is the user credentials
+# used by the Kibana server to perform maintence on the kibana_index at statup. Your Kibana
+# users will still need to authenticate with Elasticsearch (which is proxied thorugh
+# the Kibana server)
+# kibana_elasticsearch_username: user
+# kibana_elasticsearch_password: pass
+
+# If your Elasticsearch requires client certificate and key
+# kibana_elasticsearch_client_crt: /path/to/your/client.crt
+# kibana_elasticsearch_client_key: /path/to/your/client.key
+
+# If you need to provide a CA certificate for your Elasticsarech instance, put
+# the path of the pem file here.
+# ca: /path/to/your/CA.pem
+
+# The default application to load.
+default_app_id: "discover"
+
+# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to
+# request_timeout setting
+# ping_timeout: 1500
+
+# Time in milliseconds to wait for responses from the back end or elasticsearch.
+# This must be > 0
+request_timeout: 300000
+
+# Time in milliseconds for Elasticsearch to wait for responses from shards.
+# Set to 0 to disable.
+shard_timeout: 0
+
+# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying
+# startup_timeout: 5000
+
+# Set to false to have a complete disregard for the validity of the SSL
+# certificate.
+verify_ssl: true
+
+# SSL for outgoing requests from the Kibana Server (PEM formatted)
+# ssl_key_file: /path/to/your/server.key
+# ssl_cert_file: /path/to/your/server.crt
+
+# Set the path to where you would like the process id file to be created.
+# pid_file: /var/run/kibana.pid
+
+# If you would like to send the log output to a file you can set the path below.
+# This will also turn off the STDOUT log output.
+# log_file: ./kibana.log
+
+# Plugins that are included in the build, and no longer found in the plugins/ folder
+bundled_plugin_ids:
+ - plugins/dashboard/index
+ - plugins/discover/index
+ - plugins/doc/index
+ - plugins/kibana/index
+ - plugins/markdown_vis/index
+ - plugins/metric_vis/index
+ - plugins/settings/index
+ - plugins/table_vis/index
+ - plugins/vis_types/index
+ - plugins/visualize/index
+

=== added file 'files/charm/kibana.init'
--- files/charm/kibana.init	1970-01-01 00:00:00 +0000
+++ files/charm/kibana.init	2015-11-03 23:41:21 +0000
@@ -0,0 +1,32 @@
+# Kibana 4
+#
+description     "Kibana 4"
+
+start on virtual-filesystems
+stop on runlevel [06]
+
+# Respawn it if the process exits
+respawn
+
+# We're setting high here, we'll re-limit below.
+limit nofile 65550 65550
+
+setuid kibana
+setgid kibana
+
+console log
+script
+  # Defaults - Please set KB_HOME
+  PATH=/bin:/usr/bin:$PATH
+  KB_HOME=/srv/kibana4
+  KB_LOG_FILE=/var/log/kibana/kibana.log
+  KB_NICE=19
+
+  HOME="${HOME:-$KB_HOME}"
+
+  # Export variables
+  export PATH HOME
+  test -n "${JAVACMD}" && export JAVACMD
+
+  exec nice -n ${KB_NICE} ${KB_HOME}/bin/kibana --config /etc/kibana/config.yml --log-file ${KB_LOG_FILE}
+end script

=== added file 'files/charm/kibana_lb.conf'
--- files/charm/kibana_lb.conf	1970-01-01 00:00:00 +0000
+++ files/charm/kibana_lb.conf	2015-11-03 23:41:21 +0000
@@ -0,0 +1,15 @@
+upstream kibana {
+    server localhost:5601;
+}
+
+server {
+  listen *:9200;
+
+  location / {
+    # Only proxy local requests for ES.
+    allow 127.0.0.1;
+    deny all;
+    proxy_pass http://es_cluster;
+    proxy_read_timeout 90;
+  }
+}

=== modified file 'files/charm/nginx.conf'
--- files/charm/nginx.conf	2014-04-14 15:59:48 +0000
+++ files/charm/nginx.conf	2015-11-03 23:41:21 +0000
@@ -5,39 +5,9 @@
   access_log /var/log/nginx/kibana.access.log;
 
   location / {
-    root /srv/kibana3;
-    index index.html index.htm;
-  }
-
-  location ~ ^/_aliases$ {
-    proxy_pass http://es_cluster;
-    proxy_read_timeout 90;
-  }
-  location ~ ^/.*/_aliases$ {
-    proxy_pass http://es_cluster;
-    proxy_read_timeout 90;
-  }
-  location ~ ^/_nodes$ {
-    proxy_pass http://es_cluster;
-    proxy_read_timeout 90;
-  }
-  location ~ ^/.*/_search$ {
-    proxy_pass http://es_cluster;
-    proxy_read_timeout 90;
-  }
-  location ~ ^/.*/_mapping$ {
-    proxy_pass http://es_cluster;
-    proxy_read_timeout 90;
-  }
-
-   location ~ ^/kibana-int/dashboard/.*$ {
-    proxy_pass http://es_cluster;
-    proxy_read_timeout 90;
-  }
-  location ~ ^/kibana-int/temp.*$ {
-    proxy_pass http://es_cluster;
-    proxy_read_timeout 90;
-  }  
+    proxy_pass http://kibana;
+    proxy_read_timeout 90;
+  }
 }
 
 

=== modified file 'files/charm/nginx_lb.conf'
--- files/charm/nginx_lb.conf	2013-09-29 21:58:50 +0000
+++ files/charm/nginx_lb.conf	2015-11-03 23:41:21 +0000
@@ -1,3 +1,3 @@
 upstream es_cluster {
   server localhost:9200;
-}
\ No newline at end of file
+}

=== modified file 'hooks/config-changed'
--- hooks/config-changed	2015-10-28 04:09:57 +0000
+++ hooks/config-changed	2015-11-03 23:41:21 +0000
@@ -6,9 +6,8 @@
 
 sed -i "s/listen.*/listen *:$PORT;/" /etc/nginx/sites-available/kibana
 
-sed -i "s@elasticsearch: .*@elasticsearch: '//'+window.location.hostname+'$KIBANA_PATH',@" /srv/kibana3/config.js
-
 service nginx reload
+service kibana restart
 
 [[ -r .PREVOPENPORT ]] && PREVOPENPORT=$(cat .PREVOPENPORT) || PREVOPENPORT=''
 [[ -z "$PREVOPENPORT" ]] || close-port $PREVOPENPORT/tcp

=== modified file 'hooks/install'
--- hooks/install	2015-10-28 04:08:59 +0000
+++ hooks/install	2015-11-03 23:41:21 +0000
@@ -7,9 +7,7 @@
 HOME=$PWD
 KIBANA_TARBALL=files/kibana.tgz
 KIBANA_SOURCE=$(config-get kibana_source)
-KIBANA_SOURCE_UNPACK_DIRECTORY=$(config-get kibana_source_unpack_directory)
 KIBANA_SOURCE_CHECKSUM=$(config-get kibana_source_checksum)
-ELASTICSEARCH_URL=$(config-get elasticsearch_url)
 HOST=$(unit-get private-address)
 
 juju-log "install dependency"
@@ -26,29 +24,35 @@
     cp $KIBANA_TARBALL /tmp/kibana.tgz
 fi
 
+groupadd kibana
+useradd -s /bin/false -g kibana kibana
+mkdir -p /var/log/kibana
+chown kibana:kibana /var/log/kibana
+
 install -o root -g root -m 0644 files/charm/nginx.conf /etc/nginx/sites-available/kibana
 install -o root -g root -m 0644 files/charm/nginx_lb.conf /etc/nginx/sites-available/es_cluster
-
+install -o root -g root -m 0644 files/charm/kibana_lb.conf /etc/nginx/sites-available/kibana_load_balancer
+install -o root -g root -m 0644 files/charm/kibana.init /etc/init/kibana.conf
+mkdir -p /etc/kibana/
+install -o root -g root -m 0644 files/charm/config.yml /etc/kibana/kibana.conf
 pushd /srv
 
-if [[ ! -d kibana3 ]]; then
+if [[ ! -d kibana4 ]]; then
     tar xzvf /tmp/kibana.tgz
-    mv ${KIBANA_SOURCE_UNPACK_DIRECTORY} kibana3
+    # Note: charm does not currently support changing kibana source.
+    mv kibana-4.* kibana4
+    chown -R kibana:kibana kibana4
 fi
 
 rm -f /etc/nginx/sites-enabled/default
 
 popd
 
-rm -f /srv/kibana3/config.js
-install -o root -g root -m 0644 files/charm/config.js /srv/kibana3/config.js
-
-# edit the elasticsearch line to be the value of option elasticsearch_url
-sed -r -i -e 's|(^\s*elasticsearch:\s*).*$|\1'"${ELASTICSEARCH_URL}"',|' /srv/kibana3/config.js
-
 hooks/config-changed
 
 ln -s /etc/nginx/sites-available/kibana /etc/nginx/sites-enabled/kibana
 ln -s /etc/nginx/sites-available/es_cluster /etc/nginx/sites-enabled/es_cluster
+ln -s /etc/nginx/sites-available/kibana_load_balancer /etc/nginx/sites-enabled/kibana_load_balancer
 
 service nginx restart
+service kibana restart

=== modified file 'hooks/start'
--- hooks/start	2015-10-28 04:09:57 +0000
+++ hooks/start	2015-11-03 23:41:21 +0000
@@ -4,3 +4,4 @@
 # i.e. 'service apache2 start'
 
 service nginx start
+service kibana restart
\ No newline at end of file

=== modified file 'hooks/stop'
--- hooks/stop	2015-06-18 20:10:31 +0000
+++ hooks/stop	2015-11-03 23:41:21 +0000
@@ -7,5 +7,6 @@
 # rm /srv/webroot/server-live.txt && sleep 30
 
 service nginx stop
+service kibana stop
 [[ -r .PREVOPENPORT ]] && PREVOPENPORT=$(cat .PREVOPENPORT) || PREVOPENPORT=''
 [[ -z "$PREVOPENPORT" ]] || close-port $PREVOPENPORT/tcp

=== modified file 'tests/11-scale-elastic.py'
--- tests/11-scale-elastic.py	2015-03-12 18:48:08 +0000
+++ tests/11-scale-elastic.py	2015-11-03 23:41:21 +0000
@@ -50,7 +50,7 @@
 
     time.sleep(3)
 
-    url = 'http://%s/_all/_search' % kibana_unit.info['public-address']
+    url = 'http://%s:9200/_all/_search' % kibana_unit.info['public-address']
     test_dir = os.path.dirname(__file__)
     test_data_file = os.path.join(test_dir, '11-scale-elastic-query.json')
     with open(test_data_file) as fp:

=== modified file 'tests/12-port-change.py'
--- tests/12-port-change.py	2015-06-18 17:11:44 +0000
+++ tests/12-port-change.py	2015-11-03 23:41:21 +0000
@@ -8,8 +8,14 @@
 d = amulet.Deployment(series='trusty')
 
 d.add('kibana')
+d.add('elasticsearch')
+
 d.configure('kibana', {'port': 81})
 
+d.add_unit('elasticsearch', 2)
+
+d.relate('kibana:rest', 'elasticsearch:client')
+
 d.expose('kibana')
 
 try: