Merge lp:~doanac/ubuntu-ci-services-itself/restish-deb into lp:ubuntu-ci-services-itself

[Andy Doan]

restish charm: use restish .deb

ev's got this packaged now so we don't need pip for this

remove the pip dependency from the restish charm. I think this gets all our services using .deb's only

[lp:~doanac/ubuntu-ci-services-itself/restish-deb]

319. By Andy Doan on 2014-03-05

oops - must have fat-fingered "dd"

should switch to emacs, it would have taken like 15 keystrokes to
have made this mistake

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:319
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/301/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/301/rebuild

review: Approve (continuous-integration)

[Evan (ev)]

Yay! +1

We really need to centralise our use of cm.py and charmhelpers down the road. We have what, three copies of it in the tree now? :)

review: Approve

[Evan (ev)]

PS. this made my afternoon: "should switch to emacs, it would have taken like 15 keystrokes to have made this mistake"

[Chris Johnston (cjohnston)]

 merge approved

review: Approve

=== added file 'charms/precise/restish/Makefile'
--- charms/precise/restish/Makefile	1970-01-01 00:00:00 +0000
+++ charms/precise/restish/Makefile	2014-03-05 23:39:46 +0000
@@ -0,0 +1,24 @@
+PWD := $(shell pwd)
+SOURCEDEPS_DIR ?= $(shell dirname $(PWD))/.sourcecode
+PYTHON := /usr/bin/env python
+CONFIGMANAGER := $(PWD)/bin/cm.py
+
+
+sourcedeps: $(PWD)/config-manager.txt clean
+	@echo Updating source dependencies...
+	@$(PYTHON) cm.py -c $(PWD)/config-manager.txt \
+		-p $(SOURCEDEPS_DIR) \
+		-t $(PWD)
+	@$(PYTHON) build/charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \
+		-c charm-helpers.yaml \
+		-b build/charm-helpers \
+		-d hooks/charmhelpers
+	@echo Do not forget to commit the updated files if any.
+	#@cd $(PWD)/build/charm-helpers; \
+		#@$(PYTHON) setup.py install --install-purelib=$(PWD)/lib \
+			#--install-scripts=$(PWD)/lib/bin
+
+clean:
+	rm -fr hooks/charmhelpers build/charm-helpers ../.sourcecode
+
+.PHONY: sourcedeps

=== added directory 'charms/precise/restish/build'
=== added file 'charms/precise/restish/charm-helpers.yaml'
--- charms/precise/restish/charm-helpers.yaml	1970-01-01 00:00:00 +0000
+++ charms/precise/restish/charm-helpers.yaml	2014-03-05 23:39:46 +0000
@@ -0,0 +1,4 @@
+include:
+    - core
+    - fetch
+    - contrib.charmsupport

=== added file 'charms/precise/restish/cm.py'
--- charms/precise/restish/cm.py	1970-01-01 00:00:00 +0000
+++ charms/precise/restish/cm.py	2014-03-05 23:39:46 +0000
@@ -0,0 +1,172 @@
+# Copyright 2014 Canonical Ltd. All rights reserved.
+
+import os
+import re
+import sys
+import errno
+import hashlib
+import subprocess
+import optparse
+import urllib
+
+from os import curdir
+from bzrlib.branch import Branch
+from bzrlib.plugin import load_plugins
+load_plugins()
+from bzrlib.plugins.launchpad import account as lp_account
+
+if 'GlobalConfig' in dir(lp_account):
+    from bzrlib.config import LocationConfig as LocationConfiguration
+    _ = LocationConfiguration
+else:
+    from bzrlib.config import LocationStack as LocationConfiguration
+    _ = LocationConfiguration
+
+
+def get_ubunet_branch_config(config_file):
+    """
+    Retrieves the sourcedeps configuration for an ubunet source dir.
+    Returns a dict of (branch, revspec) tuples, keyed by branch name.
+    """
+    branches = {}
+    with open(config_file, 'r') as stream:
+        for line in stream:
+            line = line.split('#')[0].strip()
+            match = re.match(r'(\S+)\s+'
+                             'bzr\+ssh://([^/]+)/([^;]+)'
+                             '(?:;revno=(\d+))?', line)
+            if match:
+                name, host, branch, revno = match.group(1, 2, 3, 4)
+                if revno is None:
+                    revspec = "-1"
+                else:
+                    revspec = revno
+                branches[name] = (host, branch, revspec)
+    return branches
+
+
+def main(config_file, parent_dir, target_dir, verbose):
+    """Do the deed."""
+
+    try:
+        os.makedirs(parent_dir)
+    except OSError, e:
+        if e.errno != errno.EEXIST:
+            raise
+
+    bzr_config = LocationConfiguration(parent_dir)
+    get_lp_login = lp_account.get_lp_login
+    username = get_lp_login(bzr_config) or get_lp_login()
+    if username is None:
+        raise RuntimeError("Unable to determine launchpad login")
+    quoted_username = urllib.quote(username)
+
+    branches = sorted(get_ubunet_branch_config(config_file).iteritems())
+    for branch_name, (host, quoted_branch_spec, revspec) in branches:
+        revno = int(revspec)
+
+        # qualify mirror branch name with hash of remote repo path to deal
+        # with changes to the remote branch URL over time
+        branch_spec_digest = hashlib.sha1(quoted_branch_spec).hexdigest()
+        branch_directory = branch_spec_digest
+
+        source_path = os.path.join(parent_dir, branch_directory)
+        destination_path = os.path.join(target_dir, branch_name)
+
+        # Remove leftover symlinks/stray files.
+        try:
+            os.remove(destination_path)
+        except OSError, e:
+            if e.errno != errno.EISDIR and e.errno != errno.ENOENT:
+                raise
+
+        branch_url = ("bzr+ssh://%s@%s/%s" %
+                      (quoted_username, host, quoted_branch_spec))
+        lp_url = "lp:" + quoted_branch_spec.replace("+branch/", "")
+
+        # Create the local mirror branch if it doesn't already exist
+        if verbose:
+            sys.stderr.write('%30s: ' % (branch_name,))
+            sys.stderr.flush()
+
+        fresh = False
+        if not os.path.exists(source_path):
+            subprocess.check_call(['bzr', 'branch', '-q',
+                                   '--', branch_url, source_path])
+            fresh = True
+
+        source_branch = Branch.open(source_path)
+
+        # Freshen the source branch if required (-1 means we want tip).
+        if not fresh and (revno == -1 or revno > source_branch.revno()):
+            subprocess.check_call(['bzr', 'pull', '-q', '--overwrite', '-r',
+                                   str(revno), '-d', source_path,
+                                   '--', branch_url])
+
+        if os.path.exists(destination_path):
+            # Overwrite the destination with the appropriate revision.
+            subprocess.check_call(['bzr', 'clean-tree', '--force', '-q',
+                                   '--ignored', '-d', destination_path])
+            subprocess.check_call(['bzr', 'pull', '-q', '--overwrite',
+                                   '-r', str(revno),
+                                   '-d', destination_path, '--', source_path])
+        else:
+            # Create a new branch.
+            subprocess.check_call(['bzr', 'branch', '-q', '--hardlink',
+                                   '-r', str(revno),
+                                   '--', source_path, destination_path])
+
+        # Check the state of the destination branch.
+        destination_branch = Branch.open(destination_path)
+        destination_revno = destination_branch.revno()
+
+        if verbose:
+            sys.stderr.write('checked out %4s of %s\n' %
+                             ("r" + str(destination_revno), lp_url))
+            sys.stderr.flush()
+
+        if revno != -1 and destination_revno != revno:
+            raise RuntimeError("Expected revno %d but got revno %d" %
+                               (revno, destination_revno))
+
+if __name__ == '__main__':
+    parser = optparse.OptionParser(
+        usage="%prog [options]",
+        description=(
+            "Add a lightweight checkout in <target> for each "
+            "corresponding file in <parent>."),
+        add_help_option=False)
+    parser.add_option(
+        '-p', '--parent', dest='parent', default=None,
+        help=("The directory of the parent tree."),
+        metavar="DIR")
+    parser.add_option(
+        '-t', '--target', dest='target', default=curdir,
+        help=("The directory of the target tree."),
+        metavar="DIR")
+    parser.add_option(
+        '-c', '--config', dest='config', default=None,
+        help=("The config file to be used for config-manager."),
+        metavar="DIR")
+    parser.add_option(
+        '-q', '--quiet', dest='verbose', action='store_false',
+        help="Be less verbose.")
+    parser.add_option(
+        '-h', '--help', action='help',
+        help="Show this help message and exit.")
+    parser.set_defaults(verbose=True)
+
+    options, args = parser.parse_args()
+
+    if options.parent is None:
+        parser.error(
+            "Parent directory not specified.")
+
+    if options.target is None:
+        parser.error(
+            "Target directory not specified.")
+
+    sys.exit(main(config_file=options.config,
+                  parent_dir=options.parent,
+                  target_dir=options.target,
+                  verbose=options.verbose))

=== added file 'charms/precise/restish/config-manager.txt'
--- charms/precise/restish/config-manager.txt	1970-01-01 00:00:00 +0000
+++ charms/precise/restish/config-manager.txt	2014-03-05 23:39:46 +0000
@@ -0,0 +1,5 @@
+# After making changes to this file, to ensure that the charm payload
+# is up-to-date do:
+#
+# make
+build/charm-helpers bzr+ssh://bazaar.launchpad.net/~charm-helpers/charm-helpers/devel;revno=99

=== modified file 'charms/precise/restish/config.yaml'
--- charms/precise/restish/config.yaml	2014-02-28 21:55:28 +0000
+++ charms/precise/restish/config.yaml	2014-03-05 23:39:46 +0000
@@ -47,3 +47,28 @@
     type: int
     default: 8080
     description: "Port the application will be listening."
+
+  # Apt info used by charmhelpers
+  install_sources:
+      type: string
+      default: ""
+      description: |
+          YAML list of additional installation sources, as a string. The number
+          of install_sources must match the number of install_keys. For
+          example:
+            install_sources: |
+             - ppa:project1/ppa
+             - ppa:project2/ppa
+  install_keys:
+      type: string
+      default: ""
+      description: |
+          YAML list of GPG keys for installation sources, as a string. For apt
+          repository URLs, use the public key ID used to verify package
+          signatures. For other sources such as PPA, use empty string. This
+          list must have the same number of elements as install_sources, even
+          if the key items are all empty string. An example to go with the
+          above for install_sources:
+            install_keys: |
+             - ""
+             - ""

=== modified file 'charms/precise/restish/hooks/hooks.py'
--- charms/precise/restish/hooks/hooks.py	2014-02-28 21:55:28 +0000
+++ charms/precise/restish/hooks/hooks.py	2014-03-05 23:39:46 +0000
@@ -7,6 +7,8 @@
 import sys
 import time
 
+import charmhelpers.fetch
+
 
 def juju_info(msg):
     subprocess.check_call(['juju-log', '-l', 'INFO', msg])
@@ -83,14 +85,11 @@
 def install(config):
     pkgs = [x for x in config['packages'].split(' ') if x]
     pkgs.append('bzr')
-    pkgs.append('python-pip')  # required for restish?
+    pkgs.append('python-restish')
 
     juju_info('installing apt packages...')
-    subprocess.check_call(['apt-get', 'install', '-y', '-q'] + pkgs)
-
-    juju_info('installing restish...')
-    restish = 'restish==%s' % config['restish_version']
-    subprocess.check_call(['pip', 'install', restish])
+    charmhelpers.fetch.configure_sources(update=True)
+    charmhelpers.fetch.apt_install(pkgs)
 
     if config.get('vcs') == 'tarball':
         _install_from_tarball(config)

=== modified file 'juju-deployer/branch-source-builder.yaml.tmpl'
--- juju-deployer/branch-source-builder.yaml.tmpl	2014-03-05 11:41:47 +0000
+++ juju-deployer/branch-source-builder.yaml.tmpl	2014-03-05 23:39:46 +0000
@@ -11,6 +11,10 @@
                 # need non-default package python-amqplib for this service
                 packages: "python-webtest python-mock python-jinja2 python-amqplib"
                 json_status_path: api/v1/status
+                install_sources: |
+                    - ${CI_PPA}
+                install_keys: |
+                    - ""
         bsb-gunicorn:
             charm: gunicorn
             branch: lp:charms/precise/gunicorn@28

=== modified file 'juju-deployer/image-builder.yaml.tmpl'
--- juju-deployer/image-builder.yaml.tmpl	2014-03-05 11:20:09 +0000
+++ juju-deployer/image-builder.yaml.tmpl	2014-03-05 23:39:46 +0000
@@ -10,6 +10,10 @@
                 python_path: ./image-builder:./ci-utils
                 packages: "python-webtest python-mock python-jinja2 python-amqplib"
                 json_status_path: api/v1/status
+                install_sources: |
+                    - ${CI_PPA}
+                install_keys: |
+                    - ""
         imagebuild-gunicorn:
             charm: gunicorn
             branch: lp:charms/precise/gunicorn@28

=== modified file 'juju-deployer/lander.yaml.tmpl'
--- juju-deployer/lander.yaml.tmpl	2014-03-05 11:20:09 +0000
+++ juju-deployer/lander.yaml.tmpl	2014-03-05 23:39:46 +0000
@@ -10,6 +10,10 @@
                 python_path: "./lander:./ci-utils"
                 # need non-default package python-amqplib for this service
                 packages: "python-webtest python-mock python-jinja2 python-jenkins"
+                install_sources: |
+                    - ${CI_PPA}
+                install_keys: |
+                    - ""
         lander-gunicorn:
             charm: gunicorn
             branch: lp:charms/precise/gunicorn@28

=== modified file 'juju-deployer/test-runner.yaml.tmpl'
--- juju-deployer/test-runner.yaml.tmpl	2014-03-05 11:20:09 +0000
+++ juju-deployer/test-runner.yaml.tmpl	2014-03-05 23:39:46 +0000
@@ -23,6 +23,10 @@
                 # instance. Arguably a bug in the gunicorn charm.
 
                 packages: "python-webtest python-jinja2 python-amqplib"
+                install_sources: |
+                    - ${CI_PPA}
+                install_keys: |
+                    - ""
         tr-rabbit-worker:
             charm: rabbitmq-worker
             options: