Merge lp:~mwhudson/launchpad/no-more-devpad-ssh into lp:launchpad

[Michael Hudson-Doyle]

[Michael Hudson-Doyle (mwhudson)]

Hi Tim,

This branch uses the update-sourcecode script rather than rsync to update the sourcecode directory, which lets us remove some of the complication from the ssh setup on the instance.

The other small changes are allowing the update-sourcecode script to run before make build has run and updating the branch _before_ updating sourcecode, so that any changes to update-sourcecode in the branch being tested are applied before the script is executed.

Cheers,
mwh

[Tim Penhey (thumper)]

On Tue, 22 Sep 2009 11:40:26 Michael Hudson wrote:
> Michael Hudson has proposed merging
> lp:~mwhudson/launchpad/no-more-devpad-ssh into lp:launchpad/devel.
>
> Requested reviews:
> Tim Penhey (thumper)
>
> Hi Tim,
>
> This branch uses the update-sourcecode script rather than rsync to update
> the sourcecode directory, which lets us remove some of the complication
> from the ssh setup on the instance.
>
> The other small changes are allowing the update-sourcecode script to run
> before make build has run and updating the branch _before_ updating
> sourcecode, so that any changes to update-sourcecode in the branch being
> tested are applied before the script is executed.
>
> Cheers,
> mwh
>

Looks good to me.

  merge approved

review: Approve

=== modified file 'lib/devscripts/ec2test/builtins.py'
--- lib/devscripts/ec2test/builtins.py	2009-09-18 05:57:43 +0000
+++ lib/devscripts/ec2test/builtins.py	2009-09-26 22:37:14 +0000
@@ -393,14 +393,12 @@
         for cmd in extra_update_image_command:
             user_connection.run_with_ssh_agent(cmd)
         user_connection.run_with_ssh_agent(
-            "rsync -avp --partial --delete "
-            "--filter='P *.o' --filter='P *.pyc' --filter='P *.so' "
-            "devpad.canonical.com:/code/rocketfuel-built/launchpad/sourcecode/* "
-            "/var/launchpad/sourcecode/")
-        user_connection.run_with_ssh_agent(
             'bzr pull -d /var/launchpad/test ' + TRUNK_BRANCH)
         user_connection.run_with_ssh_agent(
             'bzr pull -d /var/launchpad/download-cache lp:lp-source-dependencies')
+        user_connection.run_with_ssh_agent(
+            "/var/launchpad/test/utilities/update-sourcecode "
+            "/var/launchpad/sourcecode")
         user_connection.close()
         root_connection = instance.connect_as_root()
         root_connection.perform(

=== modified file 'lib/devscripts/ec2test/instance.py'
--- lib/devscripts/ec2test/instance.py	2009-09-18 05:26:54 +0000
+++ lib/devscripts/ec2test/instance.py	2009-09-26 22:37:14 +0000
@@ -23,7 +23,6 @@
 
 import paramiko
 
-from devscripts.ec2test.sshconfig import SSHConfig
 from devscripts.ec2test.credentials import EC2Credentials
 
 
@@ -223,26 +222,6 @@
         as_root('sudo -u %(USER)s mkdir /home/%(USER)s/.ssh')
         root_sftp = root_connection.ssh.open_sftp()
         remote_ssh_dir = '/home/%(USER)s/.ssh' % self._vals
-        # Create config file
-        self.log('Creating %s/config\n' % (remote_ssh_dir,))
-        ssh_config_file_name = os.path.join(
-            self._vals['HOME'], '.ssh', 'config')
-        ssh_config_source = open(ssh_config_file_name)
-        config = SSHConfig()
-        config.parse(ssh_config_source)
-        ssh_config_source.close()
-        ssh_config_dest = root_sftp.open("%s/config" % remote_ssh_dir, 'w')
-        ssh_config_dest.write('CheckHostIP no\n')
-        ssh_config_dest.write('StrictHostKeyChecking no\n')
-        for hostname in ('devpad.canonical.com', 'chinstrap.canonical.com'):
-            ssh_config_dest.write('Host %s\n' % (hostname,))
-            data = config.lookup(hostname)
-            for key in ('hostname', 'gssapiauthentication', 'proxycommand',
-                        'user', 'forwardagent'):
-                value = data.get(key)
-                if value is not None:
-                    ssh_config_dest.write('    %s %s\n' % (key, value))
-        ssh_config_dest.close()
         # create authorized_keys
         self.log('Setting up %s/authorized_keys\n' % remote_ssh_dir)
         authorized_keys_file = root_sftp.open(
@@ -309,7 +288,6 @@
             finally:
                 if shutdown:
                     self.shutdown()
-        
 
     def _copy_single_file(self, sftp, local_path, remote_dir):
         """Copy `local_path` to `remote_dir` on this instance.

=== removed file 'lib/devscripts/ec2test/sshconfig.py'
--- lib/devscripts/ec2test/sshconfig.py	2009-09-11 03:26:40 +0000
+++ lib/devscripts/ec2test/sshconfig.py	1970-01-01 00:00:00 +0000
@@ -1,95 +0,0 @@
-#############################################################################
-# Modified from paramiko.config.  The change should be pushed upstream.
-# Our fork supports Host lines with more than one host.
-
-import fnmatch
-
-class SSHConfig (object):
-    """
-    Representation of config information as stored in the format used by
-    OpenSSH.  Queries can be made via L{lookup}.  The format is described in
-    OpenSSH's C{ssh_config} man page.  This class is provided primarily as a
-    convenience to posix users (since the OpenSSH format is a de-facto
-    standard on posix) but should work fine on Windows too.
-
-    @since: 1.6
-    """
-
-    def __init__(self):
-        """
-        Create a new OpenSSH config object.
-        """
-        self._config = [ { 'host': '*' } ]
-
-    def parse(self, file_obj):
-        """
-        Read an OpenSSH config from the given file object.
-
-        @param file_obj: a file-like object to read the config file from
-        @type file_obj: file
-        """
-        configs = [self._config[0]]
-        for line in file_obj:
-            line = line.rstrip('\n').lstrip()
-            if (line == '') or (line[0] == '#'):
-                continue
-            if '=' in line:
-                key, value = line.split('=', 1)
-                key = key.strip().lower()
-            else:
-                # find first whitespace, and split there
-                i = 0
-                while (i < len(line)) and not line[i].isspace():
-                    i += 1
-                if i == len(line):
-                    raise Exception('Unparsable line: %r' % line)
-                key = line[:i].lower()
-                value = line[i:].lstrip()
-
-            if key == 'host':
-                del configs[:]
-                # the value may be multiple hosts, space-delimited
-                for host in value.split():
-                    # do we have a pre-existing host config to append to?
-                    matches = [c for c in self._config if c['host'] == host]
-                    if len(matches) > 0:
-                        configs.append(matches[0])
-                    else:
-                        config = { 'host': host }
-                        self._config.append(config)
-                        configs.append(config)
-            else:
-                for config in configs:
-                    config[key] = value
-
-    def lookup(self, hostname):
-        """
-        Return a dict of config options for a given hostname.
-
-        The host-matching rules of OpenSSH's C{ssh_config} man page are used,
-        which means that all configuration options from matching host
-        specifications are merged, with more specific hostmasks taking
-        precedence.  In other words, if C{"Port"} is set under C{"Host *"}
-        and also C{"Host *.example.com"}, and the lookup is for
-        C{"ssh.example.com"}, then the port entry for C{"Host *.example.com"}
-        will win out.
-
-        The keys in the returned dict are all normalized to lowercase (look for
-        C{"port"}, not C{"Port"}.  No other processing is done to the keys or
-        values.
-
-        @param hostname: the hostname to lookup
-        @type hostname: str
-        """
-        matches = [
-            x for x in self._config if fnmatch.fnmatch(hostname, x['host'])]
-        # sort in order of shortest match (usually '*') to longest
-        matches.sort(lambda x,y: cmp(len(x['host']), len(y['host'])))
-        ret = {}
-        for m in matches:
-            ret.update(m)
-        del ret['host']
-        return ret
-
-# END paramiko config fork
-#############################################################################

=== modified file 'lib/devscripts/ec2test/testrunner.py'
--- lib/devscripts/ec2test/testrunner.py	2009-09-18 02:23:13 +0000
+++ lib/devscripts/ec2test/testrunner.py	2009-09-26 22:37:14 +0000
@@ -400,12 +400,6 @@
         user_connection = self._instance.connect_as_user()
         # Clean up the test branch left in the instance image.
         user_connection.perform('rm -rf /var/launchpad/test')
-        # get newest sources
-        user_connection.run_with_ssh_agent(
-            "rsync -avp --partial --delete "
-            "--filter='P *.o' --filter='P *.pyc' --filter='P *.so' "
-            "devpad.canonical.com:/code/rocketfuel-built/launchpad/sourcecode/* "
-            "/var/launchpad/sourcecode/")
         # Get trunk.
         user_connection.run_with_ssh_agent(
             'bzr branch %(trunk_branch)s /var/launchpad/test')
@@ -415,6 +409,10 @@
                 'cd /var/launchpad/test; bzr merge %(branch)s')
         else:
             self.log('(Testing trunk, so no branch merge.)')
+        # get newest sources
+        user_connection.run_with_ssh_agent(
+            "/var/launchpad/test/utilities/update-sourcecode "
+            "/var/launchpad/sourcecode")
         # Get any new sourcecode branches as requested
         for dest, src in self.branches:
             fulldest = os.path.join('/var/launchpad/test/sourcecode', dest)

=== modified file 'utilities/update-sourcecode'
--- utilities/update-sourcecode	2009-09-10 04:31:12 +0000
+++ utilities/update-sourcecode	2009-09-26 22:37:14 +0000
@@ -5,10 +5,12 @@
 
 """Update the sourcecode managed in sourcecode/."""
 
-import _pythonpath
-
+import os
 import sys
 
+sys.path.append(
+    os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib'))
+
 from devscripts import sourcecode