Merge lp:~james-page/charms/precise/hadoop/storage-options into lp:~charmers/charms/precise/hadoop/trunk

[James Page]

This branch adds a storage option which allows all hadoop data to be stored
in a filesystem outside of root.

This supports using /mnt in ec2 (mainly)

[Clint Byrum (clint-fewbar)]

Hi James. My only question would be whether or not we can just make it so the data is inaccessible if the value is changed, and can be made accessible again if it is changed back. "Lose all your data" sounds pretty scary.

[James Page (james-page)]

Clint

Guess you could just switch it back - description updated accordingly.

[Marc Cluet (lynxman)]

Change looks good, thank you very much James!

review: Approve

=== modified file 'config.yaml'
--- config.yaml	2012-04-24 13:23:02 +0000
+++ config.yaml	2012-06-07 13:07:25 +0000
@@ -142,3 +142,13 @@
     description: |
       The number of worker threads that for the http server. This is used for
       map output fetching.
+  hadoop.dir.base:
+    type: string
+    default: /var/lib/hadoop
+    description: |
+      The directory under which all other hadoop data is stored.  Use this
+      to take advantage of extra storage that might be avaliable.
+      .
+      You can change this in a running deployment but all existing data in
+      HDFS will be inaccessible; you can of course switch it back if you
+      do this by mistake.

=== modified file 'hooks/hadoop-common'
--- hooks/hadoop-common	2012-03-27 12:00:13 +0000
+++ hooks/hadoop-common	2012-06-07 13:07:25 +0000
@@ -159,7 +159,7 @@
     # Purge existing configuration
     rm -f $dir/1*-dfs.*
     config_element "dfs.name.dir" \
-        "/var/lib/hadoop/cache/hadoop/dfs/name" > \
+        "`config-get hadoop.dir.base`/cache/hadoop/dfs/name" > \
         $dir/10-dfs.name.dir
     config_element "dfs.namenode.handler.count" \
         "`config-get dfs.namenode.handler.count`" > \
@@ -196,13 +196,25 @@
     dir=`dotdee --dir /etc/hadoop/conf.juju/core-site.xml`
     config_basic $dir
     rm -f $dir/1*-*
-    config_element "hadoop.tmp.dir" "/var/lib/hadoop/cache/\${user.name}" > \
+    config_element "hadoop.tmp.dir" "`config-get hadoop.dir.base`/cache/\${user.name}" > \
         $dir/10-hadoop.tmp.dir
     config_element "io.file.buffer.size" "`config-get io.file.buffer.size`" > \
         $dir/11-io.file.buffer.size
     dotdee --update /etc/hadoop/conf.juju/core-site.xml || true
 }
 
+configure_tmp_dir_perms() {
+    dir=`config-get hadoop.dir.base`
+    # Make sure the directory exists
+    mkdir -p $dir/cache/hadoop
+    # We don't want to do this recursively since we may be reinstalling, in which case
+    # users have their own cache/<username> directories which shouldn't be stolen
+    chown root:hadoop $dir $dir/cache $dir/cache/hadoop
+    # Ensure group write on this directory or we can start namenode/datanode
+    chmod 775 $dir/cache/hadoop
+    chmod 1777 $dir/cache
+}
+
 configure_role_relation () {
     dir=`dotdee --dir /etc/hadoop/conf.juju/core-site.xml`
     juju-log "Configuring service unit relation $1..."
@@ -368,6 +380,7 @@
         install_base_packages
         install_optional_packages
         configure_hadoop
+        configure_tmp_dir_perms        
         ;;
     jobtracker-relation-joined)
         case $mapred_role in
@@ -541,6 +554,7 @@
     upgrade-charm|config-changed)
         install_optional_packages
         configure_hadoop
+        configure_tmp_dir_perms
         conditional_restart # only restart if pertinent config has changed!
         open_ports
         ;;

=== modified file 'revision'
--- revision	2012-04-24 13:23:02 +0000
+++ revision	2012-06-07 13:07:25 +0000
@@ -1,1 +1,1 @@
-15
+16