Merge lp:~jml/udd/deployable into lp:udd

[Jonathan Lange]

A fabtask to deploy udd to a new EC2 instance

I wanted to test out the behaviour of lp:udd with certain config changes set, but I didn't particularly want to go to the hassle of getting it running on my own machine. So, I made a script that deploys an instance to EC2.

It uses fabric to do so, and is heavily derived from the script that james_w made for lp:pkgme-service.

Some things to note:
 * it uses fixed credentials to access Launchpad. I created a user, ~jml+libdep-bot, and have stored read-only credentials for accessing Launchpad in fabtasks/lp_creds.txt
 * it doesn't install bzr-buildeb. Not sure if this is a problem.
 * it doesn't do the Apache / cricket / website configuration, mostly because I'm not interested in that stuff

Still, I think it's useful. It also might be the only existing documentation about how to deploy this thing from scratch, so that makes it handy in its own right.

[James Westby (james-w)]

Hi,

This looks good.

The credentials being in the branch aren't great, but making them read-only is
a good compromise.

It would be good to extract out the generic ec2 stuff to somewhere else, but
we should probably just live with it and expect to use juju for that before
too long.

Thanks,

James

review: Approve

=== added file 'fabfile.py'
--- fabfile.py	1970-01-01 00:00:00 +0000
+++ fabfile.py	2012-03-15 22:31:18 +0000
@@ -0,0 +1,1 @@
+from fabtasks import *

=== added directory 'fabtasks'
=== added file 'fabtasks/__init__.py'
--- fabtasks/__init__.py	1970-01-01 00:00:00 +0000
+++ fabtasks/__init__.py	2012-03-15 22:31:18 +0000
@@ -0,0 +1,1 @@
+from .deploy import *

=== added file 'fabtasks/deploy.py'
--- fabtasks/deploy.py	1970-01-01 00:00:00 +0000
+++ fabtasks/deploy.py	2012-03-15 22:31:18 +0000
@@ -0,0 +1,323 @@
+from datetime import datetime
+import os
+import subprocess
+import time
+
+from fabric.api import env, run, put
+from fabric.utils import abort
+
+from boto import ec2
+
+
+RELEASE = 'lucid'
+ARCH = 'amd64'
+FS_TYPE = 'ebs'
+INSTANCE_TYPE = 't1.micro'
+USERNAME = 'ubuntu'
+REGION_NAME = 'us-east-1'
+
+NAME_KEY = "name"
+NAME_PREFIX = "fab/udd"
+
+ALL_HOSTS = "0.0.0.0/0"
+
+
+def _get_aws_credentials():
+    """Get the AWS credentials for the user from ~/.ec2/aws_id.
+
+    :returns: a tuple of access key id, secret access key
+    """
+    key_id = None
+    secret_access_key = None
+    with open(os.path.expanduser("~/.ec2/aws_id")) as f:
+        for i, line in enumerate(f.readlines()):
+            if i == 0:
+               key_id = line.strip()
+            if i == 1:
+               secret_access_key = line.strip()
+    if key_id is None:
+        raise AssertionError("Missing key id in ~/.ec2/aws_id")
+    if secret_access_key is None:
+        raise AssertionError("Missing secret access key in ~/.ec2/aws_id")
+    return key_id, secret_access_key
+
+
+def _get_ami_id(region_name):
+    """Get the AMI to use for a particular region.
+
+    This consults the ubuntu-cloudimg-query tool to find out the best
+    AMI to use for a particular region.
+
+    :returns: the ami id (as a string)
+    """
+    proc = subprocess.Popen(
+        ['ubuntu-cloudimg-query', RELEASE, ARCH, FS_TYPE, region_name],
+        stdout=subprocess.PIPE)
+    stdout, _ = proc.communicate()
+    if proc.returncode != 0:
+        raise AssertionError("calling ubuntu-cloudimg-query failed")
+    return stdout.strip()
+
+
+def _get_security_group(conn, name):
+    security_group = conn.create_security_group(name,
+            "Access to the fab pkgme-service ec2 deployment %s" % name)
+    security_group.authorize('tcp', 22, 22, ALL_HOSTS)
+    security_group.authorize('tcp', 80, 80, ALL_HOSTS)
+    security_group.authorize('tcp', 443, 443, ALL_HOSTS)
+    return security_group
+
+
+def _new_ec2_instance(keypair):
+    """Starts a new ec2 instance, giving the specified keypair access.
+
+    This will use the AWS credentials from ~/.ec2/aws_id, and the
+    AMI recommended by ubuntu-cloudimg-query.
+
+    :returns: the boto Instance object of the launched instance. It
+        will be in a state where it can be accessed over ssh.
+    """
+    key_id, secret_access_key = _get_aws_credentials()
+    ami_id = _get_ami_id(REGION_NAME)
+    conn = ec2.connect_to_region(REGION_NAME, aws_access_key_id=key_id,
+        aws_secret_access_key=secret_access_key)
+    _remove_old_security_groups(conn)
+    image = conn.get_image(ami_id)
+    now = datetime.utcnow()
+    name = "%s/%s" % (NAME_PREFIX, now.isoformat())
+    security_group = _get_security_group(conn, name)
+    reservation = image.run(instance_type=INSTANCE_TYPE, key_name=keypair,
+            security_groups=[security_group.name])
+    instance = reservation.instances[0]
+    print("Waiting for instance %s to start..." % instance.id)
+    while True:
+        time.sleep(10)
+        instance.update()
+        if instance.state != 'pending':
+           break
+    if instance.state != 'running':
+        raise AssertionError("Instance failed to start")
+    print("Instance started as %s" % instance.dns_name)
+    print("Waiting for ssh to come up")
+    # FIXME: use something better than a sleep to determine this.
+    time.sleep(30)
+    instance.add_tag(NAME_KEY, name)
+    return instance
+
+
+def deploy_to_ec2(branch="lp:udd", use_staging_deps=False, keypair='ec2-keypair'):
+    """Deploy to a new ec2 instance.
+
+    This command will spin up an ec2 instance, and deploy to it.
+
+    To run this command you must first set up an ec2 account.
+
+        http://aws.amazon.com/ec2/
+
+    Once you have that create access keys for this to use. Go to
+
+        https://aws-portal.amazon.com/gp/aws/developer/account?ie=UTF8&action=access-key
+
+    and create an access key. Then create a file at ~/.ec2/aws_id and
+    put the "Access Key ID" on the first line, and the "Secret Access Key" on
+    the second line (with nothing else on either line.)
+
+    Next you will need an ec2 keypair. Go to the EC2 console at
+
+        https://console.aws.amazon.com/ec2/home?region=us-east-1
+
+    click on "Key Pairs" and then "Create Key Pair", name the keypair "ec2-keypair".
+    Save the resulting file as
+
+        ~/.ec2/ec2-keypair.pem
+
+    Now you are ready to deploy, so run
+
+       fab deploy_to_ec2:keypair=ec2-keypair -i ~/.ec2/ec2-keypair.pem
+
+    and wait.
+
+    Note that you will be responsible for terminating the instance after
+    use (see the destroy_ec2_instances command.)
+
+    You can also specify the following arguments:
+
+      * branch: the branch to deploy, defaults to lp:pkgme-service. To
+          test some in-progress work push to lp:~you/pkgme-service/something
+          and then specify that as the branch.
+
+      * use_staging_deps: whether to use the staging PPA for
+        dependencies, as well as the production one, defaults to False.
+
+    Arguments are all specified by attaching them to the command name, e.g.
+
+      fab deploy_to_ec2:keypair=ec2-keypair,branch=lp:~me/pkgme-service/something
+    """
+    instance = _new_ec2_instance(keypair)
+    _set_instance_as_host(instance)
+    deploy_to_existing(branch=branch, use_staging_deps=use_staging_deps)
+
+
+def _set_instance_as_host(instance):
+    """Set the host to be acted on by fab to this instance."""
+    env.host_string = "%s@%s:22" % (USERNAME, instance.dns_name)
+
+
+def deploy_to_existing(branch="lp:udd", use_staging_deps=False):
+    """Deploy to an existing instance.
+
+    This command will deploy to an existing instance. Don't use it on an
+    instance you care about as it may overwrite anything.
+
+    To specify the host to deploy to use the -H option of fab:
+
+      fab -H ubuntu@<host> deploy_to_existing
+
+    You can also specify the following arguments:
+
+      * branch: the branch to deploy, defaults to lp:pkgme-service. To
+          test some in-progress work push to lp:~you/pkgme-service/something
+          and then specify that as the branch.
+
+      * pkgme_branch: the branch of pkgme to deploy, defaults to
+        lp:pkgme.
+
+      * pkgme_binary_branch: the branch of pkgme-binary to deploy,
+        defaults to lp:pkgme-binary.
+
+      * use_staging_deps: whether to use the staging PPA for
+        dependencies, as well as the production one, defaults to False.
+
+    Arguments are all specified by attaching them to the command name, e.g.
+
+      fab deploy_to_ec2:keypair=ec2-keypair,branch=lp:~me/pkgme-service/something
+    """
+    # Don't install recommends, as IS don't either
+    run(r'echo APT::Install-Recommends "false"\; | sudo tee /etc/apt/apt.conf.d/30-no-install-recommends')
+
+    run('sudo apt-get update -q')
+    # # Upgrade the base system in case we are shipping any updates in
+    # # our PPAs
+    run('sudo apt-get dist-upgrade -q -y --force-yes')
+
+    # We don't care about postfix being configured, and don't want the
+    # debconf note on install.
+    run('echo "postfix postfix/main_mailer_type select No configuration" | sudo debconf-set-selections')
+
+    # Install the dependencies needed to get puppet going
+    # TODO: move the rest of the dependencies to puppet
+    run('sudo apt-get install -q -y --force-yes bzr apache2 python-debian')
+
+    # Create the installation area.
+    # TODO: move this to a puppet configuration
+    run('sudo useradd pkg_import')
+
+    BASE_DIR = '/srv/package-import.canonical.com/new'
+    run('sudo mkdir -p %s' % (BASE_DIR,))
+
+    # Temporarily owned by 'ubuntu' so we can do the rest of the things
+    # without root.
+    run('sudo chown -R ubuntu %s' % (BASE_DIR,))
+
+    # Grab the branches
+    # TODO: investigate re-using IS' config-manager config
+    run('bzr branch -q %s %s/scripts' % (branch, BASE_DIR))
+
+    # Exporting in an effort to save time and to thus avoid timeouts.
+    run('bzr export -q -r 6468 %s/bzr lp:bzr' % (BASE_DIR,))
+    run('bzr branch -q lp:debian/distro-info %s/distro-info' % (BASE_DIR,))
+
+    lp_creds = os.path.join(
+        os.path.dirname(os.path.abspath(__file__)), 'lp_creds.txt')
+    put(lp_creds, '%s/lp_creds.txt' % (BASE_DIR,))
+
+    run('sudo chown -R pkg_import %s' % (BASE_DIR,))
+
+    # TODO: Do much of this with puppet instead.
+    run('sudo cp %s/scripts/etc-init.d-mass-import /etc/init.d/mass-import' % (BASE_DIR,))
+    run('sudo cp %s/scripts/etc-apache2-sites-available-package-import.ubuntu.com /etc/apache2/sites-available/package-import.ubuntu.com' % (BASE_DIR,))
+    run('sudo a2ensite package-import.ubuntu.com')
+    run('sudo crontab -u pkg_import %s/scripts/importer.crontab' % (BASE_DIR,))
+    run('sudo service apache2 reload')
+    run('sudo service mass-import start')
+
+
+def _connect_to_ec2():
+    """Get a connection to ec2, using the credentials on disk."""
+    key_id, secret_access_key = _get_aws_credentials()
+    conn = ec2.connect_to_region(REGION_NAME, aws_access_key_id=key_id,
+        aws_secret_access_key=secret_access_key)
+    return conn
+
+
+def _get_ec2_instances_in_states(conn, states):
+    for reservation in conn.get_all_instances():
+        for instance in reservation.instances:
+            if instance.state in states:
+                tags = getattr(instance, "tags", {})
+                name = tags.get(NAME_KEY, None)
+                if name and name.startswith(NAME_PREFIX):
+                    yield instance
+
+
+def _get_started_ec2_instances(conn):
+    """Get the ec2 instances started from here."""
+    return _get_ec2_instances_in_states(conn, ['running'])
+
+
+def _remove_old_security_groups(conn):
+    instances = list(_get_ec2_instances_in_states(conn,
+        ['pending', 'running', 'shutting-down', 'stopping', 'stopped']))
+    instance_names = [i.tags[NAME_KEY] for i in instances]
+    groups = conn.get_all_security_groups()
+    for group in groups:
+        if not group.name.startswith(NAME_PREFIX):
+            # Not one of ours
+            continue
+        if group.name in instance_names:
+            # Still in use
+            continue
+        print("Removing old security group: %s" % group.name)
+        group.delete()
+
+
+def destroy_ec2_instances():
+    """Destroy any ec2 instances created by this deployment."""
+    conn = _connect_to_ec2()
+    for instance in _get_started_ec2_instances(conn):
+        print("Stopping %s" % instance.id)
+        instance.terminate()
+    _remove_old_security_groups(conn)
+
+
+def last_ec2_launched():
+    """Cause other commands to act on the last ec2 instance launched from here.
+
+    Use this in a list of commands to have the rest of the commands act on the
+    last ec2 instance launched with deploy_to_ec2, e.g.
+
+        fab -i ~/.ec2/ec2-keypair.pem last_ec2_launched -- ls
+
+    This will then run ls on that instance.
+    """
+    def get_date(instance):
+        return instance.tags[NAME_KEY][len(NAME_PREFIX):]
+    last_instance = None
+    conn = _connect_to_ec2()
+    _remove_old_security_groups(conn)
+    for instance in _get_started_ec2_instances(conn):
+        if last_instance is None:
+            last_instance = instance
+        else:
+            if get_date(instance) > get_date(last_instance):
+                last_instance = instance
+    if last_instance is None:
+        abort("No instances found.")
+    _set_instance_as_host(last_instance)
+
+
+def list_running_ec2_instances():
+    """List any ec2 instances that were started from here and are still running."""
+    conn = _connect_to_ec2()
+    for instance in _get_started_ec2_instances(conn):
+        print("%s: %s" % (instance.id, instance.dns_name))

=== added file 'fabtasks/lp_creds.txt'
--- fabtasks/lp_creds.txt	1970-01-01 00:00:00 +0000
+++ fabtasks/lp_creds.txt	2012-03-15 22:31:18 +0000
@@ -0,0 +1,6 @@
+[1]
+consumer_key = package-import
+consumer_secret = 
+access_token = ncTGp3RRfz4RtCLNXNXl
+access_secret = BnCSnWsKCDX4lB11bCg4drtLjQmnxdmWGg1NCljZ16XqprTntQVPCcCsPPPCDSXtk1Zljss8pltB47x2
+