Merge lp:~wgrant/launchpad/no-more-vhost-config into lp:launchpad

[William Grant]

Unmerged revisions

14141. By William Grant on 2011-10-09

Drop unneeded vhost config sections from the schema.

14140. By William Grant on 2011-10-09

Fix AppServerLayer.

14139. By William Grant on 2011-10-09

appserver config now uses a custom port. Doesn't actually work, because the layer relies on config.vhost.mainsite.rooturl.

14138. By William Grant on 2011-10-09

Comment the config schema, and use a custom portin the vhost config if it's specified.

14137. By William Grant on 2011-10-09

Inline logic slightly.

14136. By William Grant on 2011-10-09

Refactor the vhost setup code a bit.

14135. By William Grant on 2011-10-09

VirtualHostConfig now takes a port, instead of a rooturl.

14134. By William Grant on 2011-10-09

Rename the xmlrpc_private vhost to xmlrpc-private, to match the subdomain.

14133. By William Grant on 2011-10-09

Since vhost config is pretty much defined from a base domain and a port, move all that stuff into code.

14132. By William Grant on 2011-10-09

Our publisher ZCML directive now checks allvhosts, rather than config.vhosts directly.

=== modified file 'configs/development/launchpad-lazr.conf'
--- configs/development/launchpad-lazr.conf	2016-06-30 16:05:11 +0000
+++ configs/development/launchpad-lazr.conf	2016-09-24 06:42:11 +0000
@@ -94,8 +94,12 @@
 
 [launchpad]
 enable_test_openid_provider: True
+<<<<<<< TREE
 openid_canonical_root: https://testopenid.dev/
 openid_provider_root: https://testopenid.dev/
+=======
+openid_provider_root: https://testopenid.dev/
+>>>>>>> MERGE-SOURCE
 code_domain: code.launchpad.dev
 default_batch_size: 5
 max_attachment_size: 2097152
@@ -213,15 +217,24 @@
 
 [vhosts]
 use_https: True
+base_hostname: launchpad.dev
+vostok_hostname: vostok.dev
+testopenid_hostname: testopenid.dev
+enable_apidoc: True
 
 [vhost.mainsite]
-hostname: launchpad.dev
-althostnames: localhost
 openid_delegate_profile: True
 
 [vhost.api]
+<<<<<<< TREE
 hostname: api.launchpad.dev
+=======
+# Turn this on once we've solved cache invalidation problems and are
+# ready to test.
+# enable_server_side_representation_cache: True
+>>>>>>> MERGE-SOURCE
 
+<<<<<<< TREE
 [vhost.blueprints]
 hostname: blueprints.launchpad.dev
 
@@ -249,6 +262,8 @@
 [vhost.feeds]
 hostname: feeds.launchpad.dev
 
+=======
+>>>>>>> MERGE-SOURCE
 [immediate_mail]
 # XXX sinzui 2008-03-26:
 # A development box should never send email to the outer world,

=== modified file 'configs/testrunner-appserver/launchpad-lazr.conf'
--- configs/testrunner-appserver/launchpad-lazr.conf	2014-02-27 08:39:44 +0000
+++ configs/testrunner-appserver/launchpad-lazr.conf	2016-09-24 06:42:11 +0000
@@ -22,6 +22,7 @@
 xmlrpc_runner_sleep: 1
 register_bounces_every: 1
 
+<<<<<<< TREE
 [vhost.mainsite]
 rooturl: http://launchpad.dev:8085/
 
@@ -54,6 +55,10 @@
 
 [vhost.feeds]
 rooturl: http://feeds.launchpad.dev:8085/
+=======
+[vhosts]
+port: 8085
+>>>>>>> MERGE-SOURCE
 
 [immediate_mail]
 # BarryWarsaw 04-Dec-2008: AppServerLayer tests should send email to the fake

=== modified file 'configs/testrunner/launchpad-lazr.conf'
--- configs/testrunner/launchpad-lazr.conf	2016-05-18 00:33:18 +0000
+++ configs/testrunner/launchpad-lazr.conf	2016-09-24 06:42:11 +0000
@@ -107,9 +107,13 @@
 # We use the stub Google Service here which maps URL fragment to
 # to static content
 homepage_recent_posts_feed: http://launchpad.dev:8092/blog-feed
+<<<<<<< TREE
 openid_canonical_root: http://testopenid.dev/
 openid_provider_root: http://testopenid.dev/
 openid_alternate_provider_roots: http://login1.dev/, http://login2.dev/
+=======
+openid_provider_root: http://testopenid.dev/
+>>>>>>> MERGE-SOURCE
 
 [launchpad_session]
 cookie: launchpad_tests

=== added directory 'lib/devscripts/ec2test'
=== added file 'lib/devscripts/ec2test/instance.py.OTHER'
--- lib/devscripts/ec2test/instance.py.OTHER	1970-01-01 00:00:00 +0000
+++ lib/devscripts/ec2test/instance.py.OTHER	2016-09-24 06:42:11 +0000
@@ -0,0 +1,695 @@
+# Copyright 2009-2011 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Code to represent a single machine instance in EC2."""
+
+__metaclass__ = type
+__all__ = [
+    'EC2Instance',
+    ]
+
+import code
+import errno
+import glob
+import os
+import select
+import socket
+import subprocess
+import sys
+import time
+import traceback
+from datetime import datetime
+from bzrlib.errors import BzrCommandError
+from devscripts.ec2test.session import EC2SessionName
+import paramiko
+
+
+DEFAULT_INSTANCE_TYPE = 'c1.xlarge'
+AVAILABLE_INSTANCE_TYPES = ('m1.large', 'm1.xlarge', 'c1.xlarge')
+
+
+class AcceptAllPolicy:
+    """We accept all unknown host key."""
+
+    def missing_host_key(self, client, hostname, key):
+        # Normally the console output is supposed to contain the Host key but
+        # it doesn't seem to be the case here, so we trust that the host we
+        # are connecting to is the correct one.
+        pass
+
+
+def get_user_key():
+    """Get a SSH key from the agent.  Raise an error if no keys were found.
+
+    This key will be used to let the user log in (as $USER) to the instance.
+    """
+    agent = paramiko.Agent()
+    keys = agent.get_keys()
+    if len(keys) == 0:
+        raise BzrCommandError(
+            'You must have an ssh agent running with keys installed that '
+            'will allow the script to access Launchpad and get your '
+            'branch.\n')
+
+    # XXX mars 2010-05-07 bug=577118
+    # Popping the first key off of the stack can create problems if the person
+    # has more than one key in their ssh-agent, but alas, we have no good way
+    # to detect the right key to use.  See bug 577118 for a workaround.
+    return keys[0]
+
+
+# Commands to run to turn a blank image into one usable for the rest of the
+# ec2 functionality.  They come in two parts, one set that need to be run as
+# root and another that should be run as the 'ec2test' user.
+# Note that the sources from http://us.ec2.archive.ubuntu.com/ubuntu/ are per
+# instructions described in http://is.gd/g1MIT .  When we switch to
+# Eucalyptus, we can dump this.
+
+from_scratch_root = """
+# From 'help set':
+# -x  Print commands and their arguments as they are executed.
+# -e  Exit immediately if a command exits with a non-zero status.
+set -xe
+
+sed -ie 's/main universe/main universe multiverse/' /etc/apt/sources.list
+
+. /etc/lsb-release
+
+cat >> /etc/apt/sources.list << EOF
+deb http://ppa.launchpad.net/launchpad/ubuntu $DISTRIB_CODENAME main
+deb http://ppa.launchpad.net/bzr/ubuntu $DISTRIB_CODENAME main
+deb http://ppa.launchpad.net/bzr-beta-ppa/ubuntu $DISTRIB_CODENAME main
+deb http://us.ec2.archive.ubuntu.com/ubuntu/ $DISTRIB_CODENAME multiverse
+deb-src http://us.ec2.archive.ubuntu.com/ubuntu/ $DISTRIB_CODENAME main
+EOF
+
+# This next part is cribbed from rocketfuel-setup
+dev_host() {
+  sed -i \"s/^127.0.0.88.*$/&\ ${hostname}/\" /etc/hosts
+}
+
+echo 'Adding development hosts on local machine'
+echo '
+# Launchpad virtual domains. This should be on one line.
+127.0.0.88      launchpad.dev
+' >> /etc/hosts
+
+declare -a hostnames
+hostnames=$(cat <<EOF
+    answers.launchpad.dev
+    archive.launchpad.dev
+    api.launchpad.dev
+    bazaar-internal.launchpad.dev
+    blueprints.launchpad.dev
+    bugs.launchpad.dev
+    code.launchpad.dev
+    feeds.launchpad.dev
+    keyserver.launchpad.dev
+    lists.launchpad.dev
+    ppa.launchpad.dev
+    private-ppa.launchpad.dev
+    testopenid.dev
+    translations.launchpad.dev
+    xmlrpc-private.launchpad.dev
+    xmlrpc.launchpad.dev
+EOF
+    )
+
+for hostname in $hostnames; do
+  dev_host;
+done
+
+echo '
+127.0.0.99      bazaar.launchpad.dev
+' >> /etc/hosts
+
+# Add the keys for the three PPAs added to sources.list above.
+apt-key adv --recv-keys --keyserver pool.sks-keyservers.net 2af499cb24ac5f65461405572d1ffb6c0a5174af
+apt-key adv --recv-keys --keyserver pool.sks-keyservers.net ece2800bacf028b31ee3657cd702bf6b8c6c1efd
+apt-key adv --recv-keys --keyserver pool.sks-keyservers.net cbede690576d1e4e813f6bb3ebaf723d37b19b80
+
+aptitude update
+aptitude -y full-upgrade
+
+DEBIAN_FRONTEND=noninteractive apt-get -y install launchpad-developer-dependencies apache2 apache2-mpm-worker
+
+# Create the ec2test user, give them passwordless sudo.
+adduser --gecos "" --disabled-password ec2test
+echo 'ec2test\tALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+mkdir /home/ec2test/.ssh
+cat > /home/ec2test/.ssh/config << EOF
+CheckHostIP no
+StrictHostKeyChecking no
+EOF
+
+mkdir /var/launchpad
+chown -R ec2test:ec2test /var/www /var/launchpad /home/ec2test/
+"""
+
+
+from_scratch_ec2test = """
+# From 'help set':
+# -x  Print commands and their arguments as they are executed.
+# -e  Exit immediately if a command exits with a non-zero status.
+set -xe
+
+bzr launchpad-login %(launchpad-login)s
+bzr init-repo --2a /var/launchpad
+bzr branch lp:~launchpad-pqm/launchpad/devel /var/launchpad/test
+bzr branch --standalone lp:lp-source-dependencies /var/launchpad/download-cache
+mkdir /var/launchpad/sourcecode
+/var/launchpad/test/utilities/update-sourcecode /var/launchpad/sourcecode
+"""
+
+
+postmortem_banner = """\
+Postmortem Console. EC2 instance is not yet dead.
+It will shut down when you exit this prompt (CTRL-D)
+
+Tab-completion is enabled.
+EC2Instance is available as `instance`.
+Also try these:
+  http://%(dns)s/current_test.log
+  ssh -A ec2test@%(dns)s
+"""
+
+
+class EC2Instance:
+    """A single EC2 instance."""
+
+    @classmethod
+    def make(cls, name, instance_type, machine_id, demo_networks=None,
+             credentials=None):
+        """Construct an `EC2Instance`.
+
+        :param name: The name to use for the key pair and security group for
+            the instance.
+        :type name: `EC2SessionName`
+        :param instance_type: One of the AVAILABLE_INSTANCE_TYPES.
+        :param machine_id: The AMI to use, or None to do the usual regexp
+            matching.  If you put 'based-on:' before the AMI id, it is assumed
+            that the id specifies a blank image that should be made into one
+            suitable for the other ec2 functions (see `from_scratch_root` and
+            `from_scratch_ec2test` above).
+        :param demo_networks: A list of networks to add to the security group
+            to allow access to the instance.
+        :param credentials: An `EC2Credentials` object.
+        """
+        # This import breaks in the test environment.  Do it here so
+        # that unit tests (which don't use this factory) can still
+        # import EC2Instance.
+        from bzrlib.plugins.launchpad.account import get_lp_login
+
+        # XXX JeroenVermeulen 2009-11-27 bug=489073: EC2Credentials
+        # imports boto, which isn't necessarily installed in our test
+        # environment.  Doing the import here so that unit tests (which
+        # don't use this factory) can still import EC2Instance.
+        from devscripts.ec2test.credentials import EC2Credentials
+
+        assert isinstance(name, EC2SessionName)
+        if instance_type not in AVAILABLE_INSTANCE_TYPES:
+            raise ValueError('unknown instance_type %s' % (instance_type,))
+
+        # We call this here so that it has a chance to complain before the
+        # instance is started (which can take some time).
+        user_key = get_user_key()
+
+        if credentials is None:
+            credentials = EC2Credentials.load_from_file()
+
+        # Make the EC2 connection.
+        account = credentials.connect(name)
+
+        # We do this here because it (1) cleans things up and (2) verifies
+        # that the account is correctly set up. Both of these are appropriate
+        # for initialization.
+        #
+        # We always recreate the keypairs because there is no way to
+        # programmatically retrieve the private key component, unless we
+        # generate it.
+        account.collect_garbage()
+
+        if machine_id and machine_id.startswith('based-on:'):
+            from_scratch = True
+            machine_id = machine_id[len('based-on:'):]
+        else:
+            from_scratch = False
+
+        # get the image
+        image = account.acquire_image(machine_id)
+
+        login = get_lp_login()
+        if not login:
+            raise BzrCommandError(
+                'you must have set your launchpad login in bzr.')
+
+        return EC2Instance(
+            name, image, instance_type, demo_networks, account,
+            from_scratch, user_key, login)
+
+    def __init__(self, name, image, instance_type, demo_networks, account,
+                 from_scratch, user_key, launchpad_login):
+        self._name = name
+        self._image = image
+        self._account = account
+        self._instance_type = instance_type
+        self._demo_networks = demo_networks
+        self._boto_instance = None
+        self._from_scratch = from_scratch
+        self._user_key = user_key
+        self._launchpad_login = launchpad_login
+
+    def log(self, msg):
+        """Log a message on stdout, flushing afterwards."""
+        # XXX: JonathanLange 2009-05-31 bug=383076: Should delete this and use
+        # Python logging module instead.
+        sys.stdout.write(msg)
+        sys.stdout.flush()
+
+    def start(self):
+        """Start the instance."""
+        if self._boto_instance is not None:
+            self.log('Instance %s already started' % self._boto_instance.id)
+            return
+        start = time.time()
+        self.private_key = self._account.acquire_private_key()
+        self.security_group = self._account.acquire_security_group(
+            demo_networks=self._demo_networks)
+        reservation = self._image.run(
+            key_name=self._name, security_groups=[self._name],
+            instance_type=self._instance_type)
+        self._boto_instance = reservation.instances[0]
+        self.log('Instance %s starting..' % self._boto_instance.id)
+        while self._boto_instance.state == 'pending':
+            self.log('.')
+            time.sleep(5)
+            self._boto_instance.update()
+        if self._boto_instance.state == 'running':
+            self.log(' started on %s\n' % self.hostname)
+            elapsed = time.time() - start
+            self.log('Started in %d minutes %d seconds\n' %
+                     (elapsed // 60, elapsed % 60))
+            self._output = self._boto_instance.get_console_output()
+            self.log(self._output.output)
+            self._ec2test_user_has_keys = False
+        else:
+            raise BzrCommandError(
+                'failed to start: %s\n' % self._boto_instance.state)
+
+    def shutdown(self):
+        """Shut down the instance."""
+        if self._boto_instance is None:
+            self.log('no instance created\n')
+            return
+        self._boto_instance.update()
+        if self._boto_instance.state not in ('shutting-down', 'terminated'):
+            # terminate instance
+            self._boto_instance.stop()
+            self._boto_instance.update()
+        self.log('instance %s\n' % (self._boto_instance.state,))
+
+    @property
+    def hostname(self):
+        if self._boto_instance is None:
+            return None
+        return self._boto_instance.public_dns_name
+
+    def _connect(self, username):
+        """Connect to the instance as `user`. """
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(AcceptAllPolicy())
+        connect_args = {
+            'username': username,
+            'pkey': self.private_key,
+            'allow_agent': False,
+            'look_for_keys': False,
+            }
+        for count in range(10):
+            try:
+                ssh.connect(self.hostname, **connect_args)
+            except (socket.error, paramiko.AuthenticationException), e:
+                self.log('_connect: %r\n' % (e,))
+                if count < 9:
+                    time.sleep(5)
+                    self.log('retrying...')
+                else:
+                    raise
+            else:
+                break
+        return EC2InstanceConnection(self, username, ssh)
+
+    def _upload_local_key(self, conn, remote_filename):
+        """Upload a key from the local user's agent to `remote_filename`.
+
+        The key will be uploaded in a format suitable for
+        ~/.ssh/authorized_keys.
+        """
+        authorized_keys_file = conn.sftp.open(remote_filename, 'w')
+        authorized_keys_file.write(
+            "%s %s\n" % (
+                self._user_key.get_name(), self._user_key.get_base64()))
+        authorized_keys_file.close()
+
+    def _ensure_ec2test_user_has_keys(self, connection=None):
+        """Make sure that we can connect over ssh as the 'ec2test' user.
+
+        We add both the key that was used to start the instance (so
+        _connect('ec2test') works and a key from the locally running ssh agent
+        (so EC2InstanceConnection.run_with_ssh_agent works).
+        """
+        if not self._ec2test_user_has_keys:
+            if connection is None:
+                connection = self._connect('ubuntu')
+                our_connection = True
+            else:
+                our_connection = False
+            self._upload_local_key(connection, 'local_key')
+            connection.perform(
+                'cat /home/ubuntu/.ssh/authorized_keys local_key '
+                '| sudo tee /home/ec2test/.ssh/authorized_keys > /dev/null'
+                '&& rm local_key')
+            connection.perform('sudo chown -R ec2test:ec2test /home/ec2test/')
+            connection.perform('sudo chmod 644 /home/ec2test/.ssh/*')
+            if our_connection:
+                connection.close()
+            self.log(
+                'You can now use ssh -A ec2test@%s to '
+                'log in the instance.\n' % self.hostname)
+            self._ec2test_user_has_keys = True
+
+    def connect(self):
+        """Connect to the instance as a user with passwordless sudo.
+
+        This may involve first connecting as root and adding SSH keys to the
+        user's account, and in the case of a from scratch image, it will do a
+        lot of set up.
+        """
+        if self._from_scratch:
+            ubuntu_connection = self._connect('ubuntu')
+            self._upload_local_key(ubuntu_connection, 'local_key')
+            ubuntu_connection.perform(
+                'cat local_key >> ~/.ssh/authorized_keys && rm local_key')
+            ubuntu_connection.run_script(from_scratch_root, sudo=True)
+            self._ensure_ec2test_user_has_keys(ubuntu_connection)
+            ubuntu_connection.close()
+            conn = self._connect('ec2test')
+            conn.run_script(
+                from_scratch_ec2test
+                % {'launchpad-login': self._launchpad_login})
+            self._from_scratch = False
+            return conn
+        self._ensure_ec2test_user_has_keys()
+        return self._connect('ec2test')
+
+    def _report_traceback(self):
+        """Print traceback."""
+        traceback.print_exc()
+
+    def set_up_and_run(self, postmortem, shutdown, func, *args, **kw):
+        """Start, run `func` and then maybe shut down.
+
+        :param config: A dictionary specifying details of how the instance
+            should be run:
+        :param postmortem: If true, any exceptions will be caught and an
+            interactive session run to allow debugging the problem.
+        :param shutdown: If true, shut down the instance after `func` and
+            postmortem (if any) are completed.
+        :param func: A callable that will be called when the instance is
+            running and a user account has been set up on it.
+        :param args: Passed to `func`.
+        :param kw: Passed to `func`.
+        """
+        # We ignore the value of the 'shutdown' argument and always shut down
+        # unless `func` returns normally.
+        really_shutdown = True
+        retval = None
+        try:
+            self.start()
+            try:
+                retval = func(*args, **kw)
+            except Exception:
+                # When running in postmortem mode, it is really helpful to see
+                # if there are any exceptions before it waits in the console
+                # (in the finally block), and you can't figure out why it's
+                # broken.
+                self._report_traceback()
+            else:
+                really_shutdown = shutdown
+        finally:
+            try:
+                if postmortem:
+                    console = code.InteractiveConsole(locals())
+                    console.interact(
+                        postmortem_banner % {'dns': self.hostname})
+                    print 'Postmortem console closed.'
+            finally:
+                if really_shutdown:
+                    self.shutdown()
+        return retval
+
+    def _copy_single_file(self, sftp, local_path, remote_dir):
+        """Copy `local_path` to `remote_dir` on this instance.
+
+        The name in the remote directory will be that of the local file.
+
+        :param sftp: A paramiko SFTP object.
+        :param local_path: The local path.
+        :param remote_dir: The directory on the instance to copy into.
+        """
+        name = os.path.basename(local_path)
+        remote_path = os.path.join(remote_dir, name)
+        remote_file = sftp.open(remote_path, 'w')
+        remote_file.write(open(local_path).read())
+        remote_file.close()
+        return remote_path
+
+    def copy_key_and_certificate_to_image(self, sftp):
+        """Copy the AWS private key and certificate to the image.
+
+        :param sftp: A paramiko SFTP object.
+        """
+        remote_ec2_dir = '/mnt/ec2'
+        remote_pk = self._copy_single_file(
+            sftp, self.local_pk, remote_ec2_dir)
+        remote_cert = self._copy_single_file(
+            sftp, self.local_cert, remote_ec2_dir)
+        return (remote_pk, remote_cert)
+
+    def _check_single_glob_match(self, local_dir, pattern, file_kind):
+        """Check that `pattern` matches one file in `local_dir` and return it.
+
+        :param local_dir: The local directory to look in.
+        :param pattern: The glob patten to match.
+        :param file_kind: The sort of file we're looking for, to be used in
+            error messages.
+        """
+        pattern = os.path.join(local_dir, pattern)
+        matches = glob.glob(pattern)
+        if len(matches) != 1:
+            raise BzrCommandError(
+                '%r must match a single %s file' % (pattern, file_kind))
+        return matches[0]
+
+    def check_bundling_prerequisites(self, name, credentials):
+        """Check, as best we can, that all the files we need to bundle exist.
+        """
+        if subprocess.call(['which', 'ec2-register']):
+            raise BzrCommandError(
+                '`ec2-register` command not found.  '
+                'Try `sudo apt-get install ec2-api-tools`.')
+        local_ec2_dir = os.path.expanduser('~/.ec2')
+        if not os.path.exists(local_ec2_dir):
+            raise BzrCommandError(
+                "~/.ec2 must exist and contain aws_user, aws_id, a private "
+                "key file and a certificate.")
+        aws_user_file = os.path.expanduser('~/.ec2/aws_user')
+        if not os.path.exists(aws_user_file):
+            raise BzrCommandError(
+                "~/.ec2/aws_user must exist and contain your numeric AWS id.")
+        self.aws_user = open(aws_user_file).read().strip()
+        self.local_cert = self._check_single_glob_match(
+            local_ec2_dir, 'cert-*.pem', 'certificate')
+        self.local_pk = self._check_single_glob_match(
+            local_ec2_dir, 'pk-*.pem', 'private key')
+        # The bucket `name` needs to exist and be accessible. We create it
+        # here to reserve the name. If the bucket already exists and conforms
+        # to the above requirements, this is a no-op.
+        credentials.connect_s3().create_bucket(name)
+
+    def bundle(self, name, credentials):
+        """Bundle, upload and register the instance as a new AMI.
+
+        :param name: The name-to-be of the new AMI.
+        :param credentials: An `EC2Credentials` object.
+        """
+        connection = self.connect()
+        # See http://is.gd/g1MIT .  When we switch to Eucalyptus, we can dump
+        # this installation of the ec2-ami-tools.
+        connection.perform(
+            'sudo env DEBIAN_FRONTEND=noninteractive '
+            'apt-get -y  install ec2-ami-tools')
+        connection.perform('rm -f .ssh/authorized_keys')
+        connection.perform('sudo mkdir /mnt/ec2')
+        connection.perform('sudo chown $USER:$USER /mnt/ec2')
+
+        remote_pk, remote_cert = self.copy_key_and_certificate_to_image(
+            connection.sftp)
+
+        bundle_dir = os.path.join('/mnt', name)
+
+        connection.perform('sudo mkdir ' + bundle_dir)
+        connection.perform(' '.join([
+            'sudo ec2-bundle-vol',
+            '-d %s' % bundle_dir,
+            '--batch',   # Set batch-mode, which doesn't use prompts.
+            '-k %s' % remote_pk,
+            '-c %s' % remote_cert,
+            '-u %s' % self.aws_user,
+            ]))
+
+        # Assume that the manifest is 'image.manifest.xml', since "image" is
+        # the default prefix.
+        manifest = os.path.join(bundle_dir, 'image.manifest.xml')
+
+        # Best check that the manifest actually exists though.
+        test = 'test -f %s' % manifest
+        connection.perform(test)
+
+        connection.perform(' '.join([
+            'sudo ec2-upload-bundle',
+            '-b %s' % name,
+            '-m %s' % manifest,
+            '-a %s' % credentials.identifier,
+            '-s %s' % credentials.secret,
+            ]))
+
+        connection.close()
+
+        # This is invoked locally.
+        mfilename = os.path.basename(manifest)
+        manifest_path = os.path.join(name, mfilename)
+
+        env = os.environ.copy()
+        if 'JAVA_HOME' not in os.environ:
+            env['JAVA_HOME'] = '/usr/lib/jvm/default-java'
+        now = datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S UTC")
+        description = "Created %s" % now
+        cmd = [
+            'ec2-register',
+            '--private-key=%s' % self.local_pk,
+            '--cert=%s' % self.local_cert,
+            '--name=%s' % (name,),
+            '--description=%s' % description,
+            manifest_path,
+            ]
+        self.log("Executing command: %s" % ' '.join(cmd))
+        subprocess.check_call(cmd, env=env)
+
+
+class EC2InstanceConnection:
+    """An ssh connection to an `EC2Instance`."""
+
+    def __init__(self, instance, username, ssh):
+        self._instance = instance
+        self._username = username
+        self._ssh = ssh
+        self._sftp = None
+
+    @property
+    def sftp(self):
+        if self._sftp is None:
+            self._sftp = self._ssh.open_sftp()
+        return self._sftp
+
+    def perform(self, cmd, ignore_failure=False, out=None, err=None):
+        """Perform 'cmd' on server.
+
+        :param ignore_failure: If False, raise an error on non-zero exit
+            statuses.
+        :param out: A stream to write the output of the remote command to.
+        :param err: A stream to write the error of the remote command to.
+        """
+        if out is None:
+            out = sys.stdout
+        if err is None:
+            err = sys.stderr
+        self._instance.log(
+            '%s@%s$ %s\n'
+            % (self._username, self._instance._boto_instance.id, cmd))
+        session = self._ssh.get_transport().open_session()
+        session.exec_command(cmd)
+        session.shutdown_write()
+        while 1:
+            try:
+                select.select([session], [], [], 0.5)
+            except (IOError, select.error), e:
+                if e.errno == errno.EINTR:
+                    continue
+            if session.recv_ready():
+                data = session.recv(4096)
+                if data:
+                    out.write(data)
+                    out.flush()
+            if session.recv_stderr_ready():
+                data = session.recv_stderr(4096)
+                if data:
+                    err.write(data)
+                    err.flush()
+            if session.exit_status_ready():
+                break
+        session.close()
+        # XXX: JonathanLange 2009-05-31: If the command is killed by a signal
+        # on the remote server, the SSH protocol does not send an exit_status,
+        # it instead sends a different message with the number of the signal
+        # that killed the process. AIUI, this code will fail confusingly if
+        # that happens.
+        res = session.recv_exit_status()
+        if res and not ignore_failure:
+            raise RuntimeError('Command failed: %s' % (cmd,))
+        return res
+
+    def run_with_ssh_agent(self, cmd, ignore_failure=False):
+        """Run 'cmd' in a subprocess.
+
+        Use this to run commands that require local SSH credentials. For
+        example, getting private branches from Launchpad.
+        """
+        self._instance.log(
+            '%s@%s$ %s\n'
+            % (self._username, self._instance._boto_instance.id, cmd))
+        call = ['ssh', '-A', self._username + '@' + self._instance.hostname,
+               '-o', 'CheckHostIP no',
+               '-o', 'StrictHostKeyChecking no',
+               '-o', 'UserKnownHostsFile ~/.ec2/known_hosts',
+               cmd]
+        res = subprocess.call(call)
+        if res and not ignore_failure:
+            raise RuntimeError('Command failed: %s' % (cmd,))
+        return res
+
+    def run_script(self, script_text, sudo=False):
+        """Upload `script_text` to the instance and run it with bash."""
+        script = self.sftp.open('script.sh', 'w')
+        script.write(script_text)
+        script.close()
+        cmd = '/bin/bash script.sh'
+        if sudo:
+            cmd = 'sudo ' + cmd
+        self.run_with_ssh_agent(cmd)
+        # At least for mwhudson, the paramiko connection often drops while the
+        # script is running.  Reconnect just in case.
+        self.reconnect()
+        self.perform('rm script.sh')
+
+    def reconnect(self):
+        """Close the connection and reopen it."""
+        self.close()
+        self._ssh = self._instance._connect(self._username)._ssh
+
+    def close(self):
+        if self._sftp is not None:
+            self._sftp.close()
+            self._sftp = None
+        self._ssh.close()
+        self._ssh = None

=== modified file 'lib/launchpad_loggerhead/app.py'
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf	2016-09-02 18:21:07 +0000
+++ lib/lp/services/config/schema-lazr.conf	2016-09-24 06:42:11 +0000
@@ -864,6 +864,7 @@
 # datatype: boolean
 launch: True
 
+<<<<<<< TREE
 # The root URL of the canonical OpenID provider. This is used when constructing
 # OpenID URLs as user identifiers for external services such as gpgservice.
 # Since these user identifiers must remain constant forever, this setting must
@@ -878,6 +879,11 @@
 # for our accounts' OpenID identifiers. We only store the suffix, and
 # the suffix is provided by two providers.
 openid_alternate_provider_roots: none
+=======
+# The root URL of the OpenID provider used to log into Launchpad.
+# datatype: string
+openid_provider_root: none
+>>>>>>> MERGE-SOURCE
 
 # If true, the main template will be styled so that it is
 # obvious to the end user that they are using a demo system
@@ -1700,26 +1706,23 @@
 # datatype: boolean
 use_https: True
 
-[vhost.template]
-# Host name of this virtual host.
-# This is matched from the incoming Host header, and
-# also used to put together URLs if rooturl is not provided.
+# Base hostname for the Launchpad virtual hosts.
 # Example: launchpad.net
-# datatype: string
-hostname: none
-
-# Alternative host names to match, in addition to
-# the one given in hostname, comma separated.
-# Example: wwwww.launchpad.net, www.launchpad.net
-# datatype: string
-althostnames: none
-
-# Explicit root URL for this virtual host.
-# If this is not provided, the root URL is calculated
-# based on the host name.
-# Example: https://launchpad.net/
-# datatype: string
-rooturl: none
+base_hostname: none
+
+# Custom port for all virtual hosts.
+# Only used by the test suite.
+port: none
+
+# Hostnames for non-Launchpad virtual hosts.
+# Presently only used in development and tests.
+vostok_hostname: none
+testopenid_hostname: none
+
+# Should the apidoc virtual host be enabled?
+# This is the internal Zope apidoc, not public lazr.restful API docs.
+enable_apidoc: False
+
 
 [vhost.mainsite]
 # Can the profile page act as a OpenID delegated identity?
@@ -1728,6 +1731,7 @@
 
 [vhost.api]
 
+<<<<<<< TREE
 [vhost.blueprints]
 
 [vhost.code]
@@ -1749,6 +1753,11 @@
 
 # Stubbed Key server for testing purposes.  It can serve a restricted set of
 # keys in SKS format.
+=======
+
+# Stubed Key server for test proposes, it's able to serve
+# in SKS format, a restricted set of keys.
+>>>>>>> MERGE-SOURCE
 [testkeyserver]
 # Directory to be created to store the pre-installed key-files
 # datatype: string

=== modified file 'lib/lp/services/openid/adapters/openid.py'
--- lib/lp/services/openid/adapters/openid.py	2016-05-24 04:45:38 +0000
+++ lib/lp/services/openid/adapters/openid.py	2016-09-24 06:42:11 +0000
@@ -10,9 +10,26 @@
     'OpenIDPersistentIdentity',
     ]
 
+<<<<<<< TREE
 from zope.component import adapter
 from zope.interface import implementer
+=======
+from zope.component import (
+    adapter,
+    adapts,
+    )
+from zope.interface import (
+    implementer,
+    implements,
+    )
+>>>>>>> MERGE-SOURCE
 
+<<<<<<< TREE
+=======
+from canonical.config import config
+from canonical.launchpad.interfaces.account import IAccount
+from canonical.launchpad.interfaces.lpstorm import IStore
+>>>>>>> MERGE-SOURCE
 from lp.registry.interfaces.person import IPerson
 from lp.services.config import config
 from lp.services.database.interfaces import IStore
@@ -27,6 +44,7 @@
     @staticmethod
     def getServiceURL():
         """The OpenID server URL (/+openid) for the current request."""
+<<<<<<< TREE
         return config.launchpad.openid_provider_root + '+openid'
 
     @staticmethod
@@ -42,6 +60,11 @@
 
 @adapter(IAccount)
 @implementer(IOpenIDPersistentIdentity)
+=======
+        return config.openid_provider_root + '+openid'
+
+
+>>>>>>> MERGE-SOURCE
 class OpenIDPersistentIdentity:
     """A persistent OpenID identifier for a user."""
 

=== modified file 'lib/lp/services/webapp/doc/webapp-publication.txt'
--- lib/lp/services/webapp/doc/webapp-publication.txt	2016-09-21 02:50:41 +0000
+++ lib/lp/services/webapp/doc/webapp-publication.txt	2016-09-24 06:42:11 +0000
@@ -71,12 +71,24 @@
     rooturl: http://translations.launchpad.dev/
     althosts:
     ----
+<<<<<<< TREE
+=======
+    vostok @ vostok.dev
+    rooturl: http://vostok.dev/
+    althosts:
+    ----
+>>>>>>> MERGE-SOURCE
     xmlrpc @ xmlrpc.launchpad.dev
     rooturl: http://xmlrpc.launchpad.dev/
     althosts:
     ----
+<<<<<<< TREE
     xmlrpc_private @ xmlrpc-private.launchpad.dev
     rooturl: http://xmlrpc-private.launchpad.dev/
+=======
+    xmlrpc-private @ xmlrpc-private.launchpad.dev
+    rooturl: http://launchpad.dev/
+>>>>>>> MERGE-SOURCE
     althosts:
     ----
 
@@ -96,6 +108,10 @@
     localhost
     testopenid.dev
     translations.launchpad.dev
+<<<<<<< TREE
+=======
+    vostok.dev
+>>>>>>> MERGE-SOURCE
     xmlrpc-private.launchpad.dev
     xmlrpc.launchpad.dev
 

=== modified file 'lib/lp/services/webapp/login.py'
=== modified file 'lib/lp/services/webapp/metazcml.py'
--- lib/lp/services/webapp/metazcml.py	2015-10-14 15:22:01 +0000
+++ lib/lp/services/webapp/metazcml.py	2016-09-24 06:42:11 +0000
@@ -49,10 +49,15 @@
 from zope.security.proxy import ProxyFactory
 from zope.security.zcml import IPermissionDirective
 
+<<<<<<< TREE
 from lp.app.interfaces.security import IAuthorization
 from lp.layers import FeedsLayer
 from lp.services.config import config
 from lp.services.webapp.interfaces import (
+=======
+from canonical.launchpad.layers import FeedsLayer
+from canonical.launchpad.webapp.interfaces import (
+>>>>>>> MERGE-SOURCE
     IApplicationMenu,
     ICanonicalUrlData,
     IContextMenu,
@@ -60,7 +65,13 @@
     IFavicon,
     INavigationMenu,
     )
+<<<<<<< TREE
 from lp.services.webapp.publisher import RenamedView
+=======
+from canonical.launchpad.webapp.publisher import RenamedView
+from canonical.launchpad.webapp.vhosts import allvhosts
+from lp.app.interfaces.security import IAuthorization
+>>>>>>> MERGE-SOURCE
 
 
 class IAuthorizationsDirective(Interface):
@@ -617,9 +628,7 @@
     # supplied -- we don't care about the priority in Launchpad but it
     # needs to be unique -- and to do nothing if no hostname is
     # configured for this publisher.
-
-    section = getattr(config.vhost, name, None)
-    if section is None or section.hostname is None:
+    if name not in allvhosts.configs:
         return
     global _arbitrary_priority
     if priority is None:

=== modified file 'lib/lp/services/webapp/servers.py'
--- lib/lp/services/webapp/servers.py	2016-09-14 11:13:06 +0000
+++ lib/lp/services/webapp/servers.py	2016-09-24 06:42:11 +0000
@@ -1434,6 +1434,7 @@
                           ErrorReportRequest):
     """Request type for doing public XML-RPC in Launchpad."""
 
+<<<<<<< TREE
     def getRootURL(self, rootsite):
         """See IBasicLaunchpadRequest."""
         # XML-RPC requests occasionally need to use canonical_url, for
@@ -1444,6 +1445,18 @@
             rootsite = 'mainsite'
         return super(PublicXMLRPCRequest, self).getRootURL(rootsite)
 
+=======
+    def getRootURL(self, rootsite):
+        """See IBasicLaunchpadRequest."""
+        # XML-RPC requests occasionally need to use canonical_url, for
+        # the likes of sending emails. Until these are tracked down and
+        # fixed to use mainsite explicitly, replace the XML-RPC root
+        # URLs with mainsite's, so that URLs are meaningful.
+        if rootsite in (None, 'xmlrpc', 'xmlrpc-private'):
+            rootsite = 'mainsite'
+        return super(PublicXMLRPCRequest, self).getRootURL(rootsite)
+
+>>>>>>> MERGE-SOURCE
     def _createResponse(self):
         return PublicXMLRPCResponse()
 
@@ -1584,7 +1597,7 @@
 
     if private_port is not None:
         factories.append(XMLRPCRequestPublicationFactory(
-            'xmlrpc_private', PrivateXMLRPCRequest,
+            'xmlrpc-private', PrivateXMLRPCRequest,
             PrivateXMLRPCPublication, port=private_port))
 
     # Register those factories, in priority order corresponding to

=== modified file 'lib/lp/services/webapp/vhosts.py'
--- lib/lp/services/webapp/vhosts.py	2015-10-14 15:22:01 +0000
+++ lib/lp/services/webapp/vhosts.py	2016-09-24 06:42:11 +0000
@@ -3,56 +3,28 @@
 
 """Virtual host handling for the Launchpad webapp."""
 
-__all__ = ['allvhosts']
+__all__ = [
+    'allvhosts',
+    'AllVirtualHostsConfiguration',
+    ]
 
 
 class VirtualHostConfig:
     """The configuration of a single virtual host."""
 
-    def __init__(self, hostname, althostnames, rooturl, use_https):
-        if althostnames is None:
-            althostnames = []
+    def __init__(self, hostname, althostnames, use_https, port=None):
+        if use_https:
+            protocol = 'https'
         else:
-            althostnames = self._hostnameStrToList(althostnames)
-
-        if rooturl is None:
-            if use_https:
-                protocol = 'https'
-            else:
-                protocol = 'http'
-            rooturl = '%s://%s/' % (protocol, hostname)
+            protocol = 'http'
+        rooturl = (
+            '%s://%s%s/' %
+            (protocol, hostname, (':%d' % port) if port is not None else ''))
 
         self.hostname = hostname
         self.rooturl = rooturl
         self.althostnames = althostnames
 
-    @staticmethod
-    def _hostnameStrToList(althostnames):
-        """Return list of hostname strings given a string of althostnames.
-
-        This is to parse althostnames from the launchpad.conf file.
-
-        Basically, it's a comma separated list, but we're quite flexible
-        about what is accepted.  See the examples in the following doctest.
-
-        >>> thismethod = VirtualHostConfig._hostnameStrToList
-        >>> thismethod('foo')
-        ['foo']
-        >>> thismethod('foo,bar, baz')
-        ['foo', 'bar', 'baz']
-        >>> thismethod('foo,,bar, ,baz ,')
-        ['foo', 'bar', 'baz']
-        >>> thismethod('')
-        []
-        >>> thismethod(' ')
-        []
-
-        """
-        if not althostnames.strip():
-            return []
-        return [
-            name.strip() for name in althostnames.split(',') if name.strip()]
-
 
 class AllVirtualHostsConfiguration:
     """A representation of the virtual hosting configuration for
@@ -69,7 +41,7 @@
     self.hostnames : set of hostnames handled by the vhost config
     """
 
-    def __init__(self):
+    def __init__(self, config=None):
         """Initialize all virtual host settings from launchpad.conf.
 
         launchpad_conf_vhosts: The virtual_hosts config item from
@@ -77,27 +49,60 @@
 
         """
         self._has_vhost_data = False
-
+        self._config = config
+
+<<<<<<< TREE
+=======
+    def _addVHost(self, vhost, hostname, use_https=True, althostnames=None):
+        self._configs[vhost] = VirtualHostConfig(
+            hostname, althostnames or [], self._use_https and use_https,
+            self._port)
+
+>>>>>>> MERGE-SOURCE
     def _getVHostData(self):
-        """Parse the vhosts on demand."""
+        """Construct the vhosts on demand."""
         # Avoid the circular imports inherent with the use of canonical.lazr.
         if self._has_vhost_data:
             return
+<<<<<<< TREE
         from lp.services.config import config
+=======
+        if self._config is None:
+            import canonical.config
+            self._config = canonical.config.config
+        config = self._config
+>>>>>>> MERGE-SOURCE
         self._use_https = config.vhosts.use_https
+        self._port = config.vhosts.port
         self._configs = {}
         self._hostnames = set()
-        for section in config.getByCategory('vhost'):
-            if section.hostname is None:
-                continue
-            category, vhost = section.category_and_section_names
-            self._configs[vhost] = config = VirtualHostConfig(
-                section.hostname,
-                section.althostnames,
-                section.rooturl,
-                self._use_https)
-            self._hostnames.add(config.hostname)
-            self._hostnames.update(config.althostnames)
+
+        # Most vhosts are just subdomains of base_hostname.
+        normal_vhosts = [
+            'api', 'xmlrpc', 'xmlrpc-private',
+            'answers', 'blueprints', 'bugs', 'code', 'translations']
+        if config.vhosts.enable_apidoc:
+            normal_vhosts.append('apidoc')
+        for vhost in normal_vhosts:
+            self._addVHost(
+                vhost, '%s.%s' % (vhost, config.vhosts.base_hostname))
+        # mainsite is base_hostname (eg. launchpad.dev), with an optional
+        # www subdomain.
+        self._addVHost(
+            'mainsite', config.vhosts.base_hostname,
+            althostnames=['www.' + config.vhosts.base_hostname, 'localhost'])
+        # feeds never uses SSL, but is otherwise normal.
+        self._addVHost(
+            'feeds', 'feeds.' + config.vhosts.base_hostname, use_https=False)
+
+        if config.vhosts.testopenid_hostname:
+            self._addVHost('testopenid', config.vhosts.testopenid_hostname)
+        if config.vhosts.vostok_hostname:
+            self._addVHost('vostok', config.vhosts.vostok_hostname)
+
+        for conf in self._configs.itervalues():
+            self._hostnames.add(conf.hostname)
+            self._hostnames.update(conf.althostnames)
         self._has_vhost_data = True
 
     def reload(self):

=== modified file 'lib/lp/testing/layers.py'
--- lib/lp/testing/layers.py	2016-02-22 23:43:33 +0000
+++ lib/lp/testing/layers.py	2016-09-24 06:42:11 +0000
@@ -113,8 +113,35 @@
     ConfigFixture,
     ConfigUseFixture,
     )
+<<<<<<< TREE
 from lp.services.database.interfaces import IStore
 from lp.services.database.sqlbase import session_store
+=======
+from canonical.database.sqlbase import session_store
+from canonical.launchpad.scripts import execute_zcml_for_scripts
+from canonical.launchpad.webapp.interfaces import (
+    DEFAULT_FLAVOR,
+    IOpenLaunchBag,
+    IStoreSelector,
+    MAIN_STORE,
+    )
+from canonical.launchpad.webapp.servers import (
+    LaunchpadAccessLogger,
+    register_launchpad_request_publication_factories,
+    )
+import canonical.launchpad.webapp.session
+from canonical.launchpad.webapp.vhosts import AllVirtualHostsConfiguration
+from canonical.lazr import pidfile
+from canonical.lazr.testing.layers import MockRootFolder
+from canonical.lazr.timeout import (
+    get_default_timeout_function,
+    set_default_timeout_function,
+    )
+from canonical.librarian.testing.server import LibrarianServerFixture
+from canonical.testing import reset_logging
+from canonical.testing.profiled import profiled
+from canonical.testing.smtpd import SMTPController
+>>>>>>> MERGE-SOURCE
 from lp.services.googlesearch.tests.googleserviceharness import (
     GoogleServiceTestSetup,
     )
@@ -1850,7 +1877,8 @@
 
     @classmethod
     def appserver_root_url(cls):
-        return cls.appserver_config.vhost.mainsite.rooturl
+        return AllVirtualHostsConfiguration(
+            cls.appserver_config).configs['mainsite'].rooturl
 
     @classmethod
     def _waitUntilAppServerIsReady(cls):

=== modified file 'lib/lp/testing/tests/test_layers_functional.py'
--- lib/lp/testing/tests/test_layers_functional.py	2015-10-13 14:01:25 +0000
+++ lib/lp/testing/tests/test_layers_functional.py	2016-09-24 06:42:11 +0000
@@ -32,8 +32,15 @@
     getUtility,
     )
 
+<<<<<<< TREE
 from lp.services.config import config
 from lp.services.librarian.client import (
+=======
+from canonical.config import config
+from canonical.launchpad.webapp.vhosts import AllVirtualHostsConfiguration
+from canonical.lazr.pidfile import pidfile_path
+from canonical.librarian.client import (
+>>>>>>> MERGE-SOURCE
     LibrarianClient,
     UploadFailed,
     )
@@ -498,7 +505,8 @@
 
     def testAppServerIsAvailable(self):
         # Test that the app server is up and running.
-        mainsite = LayerProcessController.appserver_config.vhost.mainsite
+        mainsite = AllVirtualHostsConfiguration(
+            LayerProcessController.appserver_config).configs['mainsite']
         home_page = urlopen(mainsite.rooturl).read()
         self.failUnless(
             'Is your project registered yet?' in home_page,

=== modified file 'utilities/rocketfuel-setup'