Merge lp:~nottrobin/webteam-project-builder/restructure into lp:webteam-project-builder

[Robin Winslow]

Add "rebuild-pip-cache.sh";
Restructure to use bzr revno as build label;
create "latest" file to contain the current build label.

[lp:~nottrobin/webteam-project-builder/restructure]

6. By Robin Winslow on 2015-02-03

Spit out public interface for kelpie

7. By Robin Winslow on 2015-02-03

Make sure pip cache doesnt already exist

8. By Robin Winslow on 2015-02-03

Use the actual real echo command (remove @)

9. By Robin Winslow on 2015-02-03

Make that stuff at the end comments

10. By Robin Winslow on 2015-02-03

Add latest_file_url variable, just for out benefit

11. By Robin Winslow on 2015-02-03

Remove replacement of kelpie

12. By Robin Winslow on 2015-02-03

Actually, scratch that

13. By Robin Winslow on 2015-02-04

It'll be quicker to pull pip-cache if we can.

Unmerged revisions

13. By Robin Winslow on 2015-02-04

It'll be quicker to pull pip-cache if we can.

12. By Robin Winslow on 2015-02-03

Actually, scratch that

11. By Robin Winslow on 2015-02-03

Remove replacement of kelpie

10. By Robin Winslow on 2015-02-03

Add latest_file_url variable, just for out benefit

9. By Robin Winslow on 2015-02-03

Make that stuff at the end comments

8. By Robin Winslow on 2015-02-03

Use the actual real echo command (remove @)

7. By Robin Winslow on 2015-02-03

Make sure pip cache doesnt already exist

6. By Robin Winslow on 2015-02-03

Spit out public interface for kelpie

5. By Robin Winslow on 2015-02-02

Add "rebuild-pip-cache.sh"; Restructure to use bzr revno as build label; create "latest" file to contain the current build label.

4. By Tristram Oaten on 2015-01-15

whitespace test of bzr-sync

=== modified file '.bzrignore'
--- .bzrignore	2015-01-08 17:52:15 +0000
+++ .bzrignore	2015-02-04 00:15:03 +0000
@@ -1,3 +1,3 @@
-/build/
-/spec/
+/builds/
+/pip-caches/
 .git/

=== modified file '.gitignore'
--- .gitignore	2015-01-08 17:52:15 +0000
+++ .gitignore	2015-02-04 00:15:03 +0000
@@ -1,3 +1,3 @@
-/build/
-/spec/
+/builds/
+/pip-caches/
 .bzr/

=== modified file 'README.md'
--- README.md	2015-01-15 09:25:39 +0000
+++ README.md	2015-02-04 00:15:03 +0000
@@ -1,7 +1,8 @@
 # webteam-project-builder
 
-A script for internal Jenkins deployments at Canonical. Essentially it :
-
-Runs the `spec-repo` mojo spec with with `project-repo` python project, kicking off the `make-targets` maketarget.
-Then tarballs everything and uploads to swift, and appends the bucket location to `location-file`
-
+A script for internal Jenkins deployments at Canonical. There are two scripts:
+
+```
+./rebuild-pip-cache.sh {project-name}  # Create a cache of built pip dependencies
+./create-upload-archive.sh {project-name}  # Create a new archive of the code and upload it to swift
+```

=== removed file 'create-archive-and-update-spec.sh'
--- create-archive-and-update-spec.sh	2014-11-28 17:39:18 +0000
+++ create-archive-and-update-spec.sh	1970-01-01 00:00:00 +0000
@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-# Get arguments
-# ===
-# Example:
-# ./create-archive-and-update-spec.sh \
-#     --spec-repo=lp:~webteam-backend/canonical-mojo-specs/mojo-webteam-assets \
-#     --project-name=assets-manager \
-#     --project-repo=lp:assets-manager \
-#     --make-targets=pip-cache
-#     --location-file=common/scripts/manager-archive-location.cfg
-
-PARSED_OPTIONS=$(getopt -n "$0"  -o "s:,n:,r:,m:,l:" --long "spec-repo:,project-name:,project-repo:,make-targets:,location-file:"  -- "$@")
-eval set -- "$PARSED_OPTIONS"
-
-# extract options and their arguments into variables.
-while true ; do
-    case "$1" in
-        -s|--spec-repo)     spec_repo=$2;     shift 2;;
-        -n|--project-name)  project_name=$2;  shift 2;;
-        -r|--project-repo)  project_repo=$2;  shift 2;;
-        -m|--make-targets)  make_targets=$2;  shift 2;;
-        -l|--location-file) location_file=$2; shift 2;;
-        --) shift ; break;;
-        *) echo "Internal error!" ; exit 1;;
-    esac
-done
-
-# Constants
-project_dir=build/${project_name}
-archive_filename=${project_name}.tar.gz
-archive_path=build/${archive_filename}
-iso_timestamp=$(date --iso-8601=seconds)
-
-# Make sure build directory exists
-mkdir -p build
-
-# Checkout project
-if [ -d ${project_dir} ]; then
-    bzr pull --directory ${project_dir} --overwrite ${project_repo}
-else
-    bzr branch ${project_repo} ${project_dir}
-fi
-
-# Prepare project
-make -C ${project_dir} ${make_targets}  # Run any necessary make targets
-
-# Create archive
-rm -f ${archive_path}
-tar -C ${project_dir} --exclude-vcs -czf ${archive_path} .
-
-# Upload to swift container
-swift upload charm-assets ${archive_path} --object-name=${iso_timestamp}/${archive_filename}
-# Make sure the container is publicly accessible
-swift post -r .r:* charm-assets
-# Get the URL
-container_url=`swift stat -v charm-assets | grep -o 'http.*'`
-
-# Checkout spec
-if [ -d spec ]; then
-    bzr pull --directory spec --overwrite ${spec_repo}
-else
-    bzr branch ${spec_repo} spec
-fi
-
-cd spec
-
-# Update manager-archive-location.sh with the new archive location
-echo "CODE_ASSETS_URI=${container_url}" > ${location_file}
-echo "BUILD_LABEL=${iso_timestamp}" >> ${location_file}
-echo "ARCHIVE_FILENAME=${archive_filename}" >> ${location_file}
-
-bzr add ${location_file}
-bzr commit ${location_file} -m "Jenkins: updating manager's archive-location for release"
-bzr push ${spec_repo}

=== added file 'create-upload-archive.sh'
--- create-upload-archive.sh	1970-01-01 00:00:00 +0000
+++ create-upload-archive.sh	2015-02-04 00:15:03 +0000
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Get project name
+if [ -z "${project_name}" ]; then
+    if [ -n "${1}" ]; then
+        project_name=${1}
+    else
+        echo "Usage: ./rebuild-pip-cache.sh {project_name}";
+        exit;
+    fi
+fi
+
+# Get arguments
+# ===
+# Examples:
+# ./create-archive-and-update-spec.sh assets-manager  # project-repository and swift-container will be inferred
+# ./create-archive-and-update-spec.sh assets-manager --make-targets sass  # Optionally run make target
+# ./create-archive-and-update-spec.sh assets-manager --project-repository lp:assets-manager --pip-cache-repository lp:~webteam-backend/assets-maanager/pip-cache --swift-container assets-manager --make-targets sass
+
+PARSED_OPTIONS=$(getopt -n "$0"  -o "r:,p:,m:" --long "project-repository:,pip-cache-repository:,make-targets:"  -- "$@")
+eval set -- "$PARSED_OPTIONS"
+
+# extract options and their arguments into variables.
+while true ; do
+    case "$1" in
+        -r|--project-repository)   project_repository=$2;   shift 2;;
+        -p|--pip-cache-repository) pip_cache_repository=$2; shift 2;;
+        -m|--make-targets)         make_targets=$2;         shift 2;;
+        --) shift; break;;
+        *) echo "Error: Option parsing failure" ; exit 1;;
+    esac
+done
+
+# Infer variables from project_name
+if [ -z "${project_repository}" ];    then project_repository=lp:${project_name}; fi
+if [ -z "${pip_cache_repository}" ];  then pip_cache_repository=lp:~webteam-backend/${project_name}/pip-cache; fi
+
+# Constants
+archive_filename=${project_name}.tar.gz
+
+# Make sure builds directory exists
+mkdir -p builds
+cd builds
+
+# Checkout project
+if [ -d ${project_name} ]; then
+    bzr pull --directory ${project_name} --overwrite ${project_repository}
+else
+    bzr branch ${project_repository} ${project_name}
+fi
+
+# Get pip-cache
+if [ -d ${project_name}/pip-cache} ]; then
+    bzr pull --directory ${project_name}/pip-cache --overwrite ${pip_cache_repository}
+else
+    bzr branch ${pip_cache_repository} ${project_name}/pip-cache 
+fi
+
+# Run make targets
+if [ -n "${make_targets}" ]; then
+    make -C ${project_name} ${make_targets}  # Run any necessary make targets
+fi
+
+# Create archive
+rm -f ${archive_filename}
+tar -C ${project_name} --exclude-vcs -czf ${archive_filename} .
+
+# Get revision
+latest_revision=$(bzr revno ${project_name})
+
+# Upload to swift container
+swift upload ${project_name} ${archive_filename} --object-name=${latest_revision}/${archive_filename}
+# Make sure the container is publicly accessible
+swift post -r .r:* ${project_name}
+
+# Save the latest build label
+echo ${latest_revision} > latest
+swift upload ${project_name} latest
+
+# Get the URL
+container_url=`swift stat -v ${project_name} | grep -o 'http.*'`
+# Replace kelpie with external interface
+container_url=$(echo ${container_url} | sed "s/http:\/\/kelpie.internal:8080/https:\/\/objectstorage.prodstack.canonical.com/")
+archive_url="${container_url}/${latest_revision}/${archive_filename}"
+latest_file_url=${container_url}/latest
+
+# ===
+# Latest revision:  ${latest_revision}
+# URL for 'latest': ${latest_file_url}
+# URL for archive:  ${archive_url}
+# ===

=== added file 'rebuild-pip-cache.sh'
--- rebuild-pip-cache.sh	1970-01-01 00:00:00 +0000
+++ rebuild-pip-cache.sh	2015-02-04 00:15:03 +0000
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Get project name
+if [ -z "${project_name}" ]; then
+    if [ -n "${1}" ]; then
+        project_name=${1}
+    else
+        echo "Usage: ./rebuild-pip-cache.sh {project_name}";
+        exit;
+    fi
+fi
+
+# Get arguments
+# ===
+# Example:
+# ./rebuild-pip-cache.sh assets-mapper  # project-repo and pip-cache-repo will be inferred
+# ./rebuild-pip-cache.sh assets-mapper -r lp:assets-manager -c lp:~webteam-backend/assets-manager/pip-cache
+# ./rebuild-pip-cache.sh assets-manager --project-repo=lp:assets-manager --pip-cache-repo=lp:~webteam-backend/assets-manager/pip-cache
+
+PARSED_OPTIONS=$(getopt -n "$0"  -o "r:,p:,:c" --long "project-repo:,pip-cache-repo:,create"  -- "$@")
+eval set -- "$PARSED_OPTIONS"
+
+# extract options and their arguments into variables.
+while true ; do
+    case "$1" in
+        -r|--project-repo)   project_repo=$2;   shift 2;;
+        -p|--pip-cache-repo) pip_cache_repo=$2; shift 2;;
+        -c|--create) create=true; shift;;
+        --) shift; break;;
+        *) echo "Error: Option parsing failure"; exit 1;;
+    esac
+done
+
+# Infer variables from project_name
+if [ -z "${project_repo}" ];   then project_repo=lp:${project_name}; fi
+if [ -z "${pip_cache_repo}" ]; then pip_cache_repo=lp:~webteam-backend/${project_name}/pip-cache; fi
+
+# Create project directory in pip-caches
+mkdir -p pip-caches/${project_name}
+cd pip-caches/${project_name}
+
+# Get pip-cache
+if [ -d pip-cache ]; then
+    bzr pull --directory pip-cache --overwrite ${pip_cache_repo}
+else
+    # Try-catch
+    {
+        bzr branch ${pip_cache_repo} pip-cache  # Create pip-cache
+    } || {
+        if [ ! ${create} ]; then
+            echo "repository ${pip_cache_repo} doesn't exist";
+            exit
+        else
+            # Create blank pip-cache repo
+            mkdir pip-cache
+            bzr init pip-cache
+        fi
+    }
+fi
+
+# Get project
+if [ -d code ]; then
+    bzr pull --directory code --overwrite ${project_repo}
+else
+    bzr branch ${project_repo} code
+fi
+
+# Build pip-cache
+pip install --exists-action=w --download pip-cache/ -r code/requirements/standard.txt
+bzr add pip-cache/.
+bzr commit pip-cache/ --unchanged -m 'Requirements auto-updated by webteam-project-builder'
+bzr push --directory pip-cache ${pip_cache_repo}