Merge lp:~zulcss/vmbuilder/vmbuilder-ec2-rewrite into lp:vmbuilder/trunk

[Chuck Short]

[Chuck Short (zulcss)]

New ec2 crack with suggested changes.

[Soren Hansen (soren)]

On Wed, May 13, 2009 at 05:35:14PM -0000, Chuck Short wrote:
> Chuck Short has proposed merging lp:~zulcss/vmbuilder/vmbuilder-ec2-rewrite into lp:vmbuilder.
>
> Requested reviews:
> Soren Hansen (soren)
>
> New ec2 crack with suggested changes.
> --
> https://code.launchpad.net/~zulcss/vmbuilder/vmbuilder-ec2-rewrite/+merge/6536
> You are requested to review the proposed merge of lp:~zulcss/vmbuilder/vmbuilder-ec2-rewrite into lp:vmbuilder.

> === modified file 'VMBuilder/plugins/ec2/__init__.py'
> --- VMBuilder/plugins/ec2/__init__.py 2009-05-04 08:32:46 +0000
> +++ VMBuilder/plugins/ec2/__init__.py 2009-05-13 15:20:12 +0000
> @@ -17,8 +17,9 @@
> # You should have received a copy of the GNU General Public License
> # along with this program. If not, see <http://www.gnu.org/licenses/>.
> #
> -from VMBuilder import register_plugin, Plugin, VMBuilderUserError
> +from VMBuilder import register_plugin, Plugin, VMBuilderUserError, VMBuilderException
> from VMBuilder.util import run_cmd
> +import VMBuilder
> import logging
> import os
>
> @@ -37,12 +38,23 @@
> group.add_option('--ec2-secret-key', metavar='SECRET_ID', help='AWS secret access key.')
> group.add_option('--ec2-kernel','--ec2-aki', metavar='AKI', help='EC2 AKI (kernel) to use.')
> group.add_option('--ec2-ramdisk','--ec2-ari', metavar='ARI', help='EC2 ARI (ramdisk) to use.')
> + group.add_option('--ec2-version',metavar='EC2_VER', help='Specifity the EC2 image version.')
> + group.add_option('--ec2-landscape',action='store_true', help='Install landscape client support')
> self.vm.register_setting_group(group)
>
> def preflight_check(self):
> if not self.vm.ec2:
> return True
>
> + try:
> + run_cmd('ec2-ami-tools-version')
> + except VMBuilderException, e:
> + raise VMBuilderUserError('You need to have the Amazon EC2 AMI tools installed')
> +

This part is fine.

> + modname = 'VMBuilder.plugins.ubuntu.%s' % (self.vm.suite, )
> + mod = __import__(modname, fromlist=[self.vm.suite])
> + self.suite = getattr(mod, self.vm.suite.capitalize())(self.vm)
> +

This part isn't. Ideally, the EC2 plugin shouldn't know which distro it's
deploying. I realise this is going to be difficult at this point (given the
landscape integration and all that), but there's absolutely no reason to have
the EC2 plugin load the suite module from the Ubuntu plugin.

As I said when we discussed this: Split out that stuff from the Ubuntu plugin's
preflight_check, and put it into a function by itself, which can be called
whenever something needs the suite. For instance, you can replace each call in
the Ubuntu plugin to self.suite to a call to self.suite() which will take care
of the initialisation. At any rate, this does not belong in the EC2 plugin.

> + 'screen-profiles']

I didn't answer my question about whether screen-profiles overrides the screen
command by default?

> - self.vm.ppa += ['ubuntu-ec2']
> + self.vm.ppa += ['ubuntu--on-ec2']

Two dashes? Really?

>
> def post_install(self):
> if not self.vm....

review: Needs Fixing

Unmerged revisions

=== modified file 'VMBuilder/plugins/ec2/__init__.py'
--- VMBuilder/plugins/ec2/__init__.py	2009-05-04 08:32:46 +0000
+++ VMBuilder/plugins/ec2/__init__.py	2009-05-13 15:20:12 +0000
@@ -17,8 +17,9 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
-from VMBuilder import register_plugin, Plugin, VMBuilderUserError
+from VMBuilder import register_plugin, Plugin, VMBuilderUserError, VMBuilderException
 from VMBuilder.util import run_cmd
+import VMBuilder
 import logging
 import os
 
@@ -37,12 +38,23 @@
         group.add_option('--ec2-secret-key', metavar='SECRET_ID', help='AWS secret access key.')
         group.add_option('--ec2-kernel','--ec2-aki', metavar='AKI', help='EC2 AKI (kernel) to use.')
         group.add_option('--ec2-ramdisk','--ec2-ari', metavar='ARI', help='EC2 ARI (ramdisk) to use.')
+        group.add_option('--ec2-version',metavar='EC2_VER', help='Specifity the EC2 image version.')
+        group.add_option('--ec2-landscape',action='store_true', help='Install landscape client support')
         self.vm.register_setting_group(group)
 
     def preflight_check(self):
         if not self.vm.ec2:
             return True
 
+        try:
+            run_cmd('ec2-ami-tools-version')
+        except VMBuilderException, e:
+            raise VMBuilderUserError('You need to have the Amazon EC2 AMI tools installed')
+
+        modname = 'VMBuilder.plugins.ubuntu.%s' % (self.vm.suite, )
+        mod = __import__(modname, fromlist=[self.vm.suite])
+        self.suite = getattr(mod, self.vm.suite.capitalize())(self.vm)
+
         if not self.vm.hypervisor.name == 'Xen':
             raise VMBuilderUserError('When building for EC2 you must use the xen hypervisor.')
 
@@ -66,17 +78,13 @@
 
         if not self.vm.ec2_kernel:
             logging.debug('No ec2-aki choosen setting to default. Use --ec2-kernel to change this')
-            if self.vm.arch == 'amd64':
-                self.vm.ec2_kernel = 'aki-d314f0ba'
-            else:
-                self.vm.ec2_kernel = 'aki-af14f0c6'
+            self.vm.ec2_kernel = self.suite.ec2_kernel_info[self.vm.arch]
+        logging.debug('%s - to be used for AKI.' %(self.vm.ec2_kernel))
 
         if not self.vm.ec2_ramdisk:
             logging.debug('No ec2-ari choosen setting to default. Use --ec2-ramdisk to change this.')
-            if self.vm.arch == 'amd64':
-                self.vm.ec2_ramdisk = 'ari-d014f0b9'
-            else:
-                self.vm.ec2_ramdisk = 'ari-ac14f0c5'
+            self.vm.ec2_ramdisk = self.suite.ec2_ramdisk_info[self.vm.arch]
+        logging.debug('%s - to be used for ARI.' %(self.vm.ec2_ramdisk))
 
         if not self.vm.ec2_bucket:
             raise VMBuilderUserError('When building for EC2 you must provide an S3 bucket to hold the AMI')
@@ -86,30 +94,52 @@
 
         if not self.vm.ec2_secret_key:
             raise VMBuilderUserError('When building for EC2 you must provide your AWS secret access key.')
+        if not self.vm.ec2_version:
+            raise VMBuilderUserError('When building for EC2 you must provide version info.')
 
 
         if not self.vm.addpkg:
              self.vm.addpkg = []
 
-        self.vm.addpkg += ['openssh-server']
-        self.vm.addpkg += ['ec2-init']
-        self.vm.addpkg += ['openssh-server']
-        self.vm.addpkg += ['ec2-modules']
-        self.vm.addpkg += ['server^']
-        self.vm.addpkg += ['standard^']
+        self.vm.addpkg += ['ec2-init',
+                          'openssh-server',
+                          'ec2-modules',
+                          'standard^',
+                          'ec2-ami-tools',
+                          'update-motd',
+                          'screen',
+                          'screen-profiles']
+
+        if self.vm.ec2_landscape:
+            logging.info('Installing landscape support')
+            self.vm.addpkg += ['landscape-common','landscape-client']
 
         if not self.vm.ppa:
             self.vm.ppa = []
 
-        self.vm.ppa += ['ubuntu-ec2']
+        self.vm.ppa += ['ubuntu--on-ec2']
 
     def post_install(self):
         if not self.vm.ec2:
             return
 
         logging.info("Running ec2 postinstall")
-        self.install_from_template('/etc/event.d/xvc0', 'upstart')
-        self.run_in_target('passwd', '-l', self.vm.user)
+        self.install_from_template('/etc/ec2_version', 'ec2_version', { 'version' : self.vm.ec2_version } )
+        self.install_from_template('/etc/ssh/sshd_config', 'sshd_config')
+        self.install_from_template('/etc/sudoers', 'sudoers')
+
+        if self.vm.suite == 'hardy':
+            self.install_from_template('/etc/update-motd.d/51_update-motd', '51_update-motd-hardy')
+        else:
+             self.install_from_template('/etc/update-motd.d/51_update-motd', '51_update-motd')
+        self.run_in_target('chmod', '755', '/etc/update-motd.d/51_update-motd')
+
+        if self.vm.ec2_landscape:
+            self.install_from_template('/etc/default/landscape-client', 'landscape_client')
+
+        self.suite.install_ec2()
+        self.run_in_target('update-rc.d', '-f', 'hwclock.sh', 'remove')
+        self.run_in_target('chpasswd', '-e', stdin='%s!\n' (self.vm.user))
 
     def deploy(self):
         if not self.vm.ec2:

=== added file 'VMBuilder/plugins/ec2/templates/51_update-motd-hardy.tmpl'
--- VMBuilder/plugins/ec2/templates/51_update-motd-hardy.tmpl	1970-01-01 00:00:00 +0000
+++ VMBuilder/plugins/ec2/templates/51_update-motd-hardy.tmpl	2009-05-12 19:38:49 +0000
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+echo "---------------------------------------------------------------------"
+echo "At the moment, only the core of the system is installed. To tune the "
+echo "system to your needs, you can choose to install one or more          "
+echo "predefined collections of software by running the following          "
+echo "command:                                                             "
+echo "                                                                     "
+echo "   sudo tasksel                                                      "
+echo "---------------------------------------------------------------------"

=== added file 'VMBuilder/plugins/ec2/templates/51_update-motd.tmpl'
--- VMBuilder/plugins/ec2/templates/51_update-motd.tmpl	1970-01-01 00:00:00 +0000
+++ VMBuilder/plugins/ec2/templates/51_update-motd.tmpl	2009-05-12 19:38:49 +0000
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+echo "---------------------------------------------------------------------"
+echo "At the moment, only the core of the system is installed. To tune the "
+echo "system to your needs, you can choose to install one or more          "
+echo "predefined collections of software by running the following          "
+echo "command:                                                             "
+echo "                                                                     "
+echo "   sudo tasksel --section server                                     "
+echo "---------------------------------------------------------------------"

=== added file 'VMBuilder/plugins/ec2/templates/ec2_version.tmpl'
--- VMBuilder/plugins/ec2/templates/ec2_version.tmpl	1970-01-01 00:00:00 +0000
+++ VMBuilder/plugins/ec2/templates/ec2_version.tmpl	2009-05-12 19:25:52 +0000
@@ -0,0 +1,1 @@
+$version

=== added file 'VMBuilder/plugins/ec2/templates/sshd_config.tmpl'
--- VMBuilder/plugins/ec2/templates/sshd_config.tmpl	1970-01-01 00:00:00 +0000
+++ VMBuilder/plugins/ec2/templates/sshd_config.tmpl	2009-05-12 19:38:49 +0000
@@ -0,0 +1,77 @@
+# Package generated configuration file
+# See the sshd(8) manpage for details
+
+# What ports, IPs and protocols we listen for
+Port 22
+# Use these options to restrict which interfaces/protocols sshd will bind to
+#ListenAddress ::
+#ListenAddress 0.0.0.0
+Protocol 2
+# HostKeys for protocol version 2
+HostKey /etc/ssh/ssh_host_rsa_key
+HostKey /etc/ssh/ssh_host_dsa_key
+#Privilege Separation is turned on for security
+UsePrivilegeSeparation yes
+
+# Lifetime and size of ephemeral version 1 server key
+KeyRegenerationInterval 3600
+ServerKeyBits 768
+
+# Logging
+SyslogFacility AUTH
+LogLevel INFO
+
+# Authentication:
+LoginGraceTime 120
+PermitRootLogin yes
+StrictModes yes
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+#AuthorizedKeysFile	%h/.ssh/authorized_keys
+
+# Don't read the user's ~/.rhosts and ~/.shosts files
+IgnoreRhosts yes
+# For this to work you will also need host keys in /etc/ssh_known_hosts
+RhostsRSAAuthentication no
+# similar for protocol version 2
+HostbasedAuthentication no
+# Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication
+#IgnoreUserKnownHosts yes
+
+# To enable empty passwords, change to yes (NOT RECOMMENDED)
+PermitEmptyPasswords no
+
+# Change to yes to enable challenge-response passwords (beware issues with
+# some PAM modules and threads)
+ChallengeResponseAuthentication no
+
+# Change to no to disable tunnelled clear text passwords
+PasswordAuthentication no
+
+# Kerberos options
+#KerberosAuthentication no
+#KerberosGetAFSToken no
+#KerberosOrLocalPasswd yes
+#KerberosTicketCleanup yes
+
+# GSSAPI options
+#GSSAPIAuthentication no
+#GSSAPICleanupCredentials yes
+
+X11Forwarding yes
+X11DisplayOffset 10
+PrintMotd no
+PrintLastLog yes
+TCPKeepAlive yes
+#UseLogin no
+
+#MaxStartups 10:30:60
+#Banner /etc/issue.net
+
+# Allow client to pass locale environment variables
+AcceptEnv LANG LC_*
+
+Subsystem sftp /usr/lib/openssh/sftp-server
+
+UsePAM yes

=== added file 'VMBuilder/plugins/ec2/templates/sudoers.tmpl'
--- VMBuilder/plugins/ec2/templates/sudoers.tmpl	1970-01-01 00:00:00 +0000
+++ VMBuilder/plugins/ec2/templates/sudoers.tmpl	2009-05-12 19:38:49 +0000
@@ -0,0 +1,23 @@
+# /etc/sudoers
+#
+# This file MUST be edited with the 'visudo' command as root.
+#
+# See the man page for details on how to write a sudoers file.
+#
+
+Defaults	env_reset
+
+# Uncomment to allow members of group sudo to not need a password
+# %sudo ALL=NOPASSWD: ALL
+
+# Host alias specification
+
+# User alias specification
+
+# Cmnd alias specification
+
+# User privilege specification
+root	ALL=(ALL) ALL
+ubuntu  ALL=(ALL) NOPASSWD:ALL
+
+# Members of the admin group may gain root privileges

=== removed file 'VMBuilder/plugins/ec2/templates/upstart.tmpl'
--- VMBuilder/plugins/ec2/templates/upstart.tmpl	2008-11-11 18:28:57 +0000
+++ VMBuilder/plugins/ec2/templates/upstart.tmpl	1970-01-01 00:00:00 +0000
@@ -1,16 +0,0 @@
-# tty1 - getty
-#
-# This service maintains a getty on tty1 from the point the system is
-# started until it is shut down again.
-
-start on stopped rc2
-start on stopped rc3
-start on stopped rc4
-start on stopped rc5
-
-stop on runlevel 0
-stop on runlevel 1
-stop on runlevel 6
-
-respawn
-exec /sbin/getty 38400 xvc0

=== modified file 'VMBuilder/plugins/ubuntu/dapper.py'
--- VMBuilder/plugins/ubuntu/dapper.py	2009-05-07 21:00:20 +0000
+++ VMBuilder/plugins/ubuntu/dapper.py	2009-05-13 15:24:34 +0000
@@ -94,6 +94,9 @@
         logging.debug("Copy host settings")
         self.copy_settings()
 
+        logging.debug("Setting timezone to UTC")
+        self.set_utc_timezone()
+
         logging.debug("Making sure system is up-to-date")
         self.update()
 
@@ -136,8 +139,15 @@
         else:
             self.run_in_target('chpasswd', '-e', stdin='root:!\n')
 
+        if self.vm.lock:
+            logging.info('Locking %s' %(self.vm.user))
+            self.run_in_target('chpasswd', '-e', stdin=('%s:!\n' %(self.vm.user)))
+
     def create_initial_user(self):
-        self.run_in_target('adduser', '--disabled-password', '--gecos', self.vm.name, self.vm.user)
+        if self.vm.id:
+            self.run_in_target('adduser', '--disabled-password', '--uid', self.vm.id, '--gecos', self.vm.name, self.vm.user)
+        else:
+            self.run_in_target('adduser', '--disabled-password', '--gecos', self.vm.name, self.vm.user)
         self.run_in_target('addgroup', '--system', 'admin')
         self.run_in_target('adduser', self.vm.user, 'admin')
 
@@ -318,3 +328,8 @@
 
     def install_vmbuilder_log(self, logfile, rootdir):
         shutil.copy(logfile, '%s/var/log/vmbuilder-install.log' % (rootdir,))
+
+    def set_utc_timezone(self):
+        if self.vm.set_utc:
+            self.run_in_target('rm', '-f', '/etc/localtime')
+            self.run_in_target('cp', '-a', '/usr/share/zoneinfo/UTC', '/etc/localtime')

=== modified file 'VMBuilder/plugins/ubuntu/distro.py'
--- VMBuilder/plugins/ubuntu/distro.py	2009-05-06 07:15:11 +0000
+++ VMBuilder/plugins/ubuntu/distro.py	2009-05-13 15:24:34 +0000
@@ -63,6 +63,7 @@
         group.add_option('--components', metavar='COMPS', help='A comma seperated list of distro components to include (e.g. main,universe).')
         group.add_option('--ppa', metavar='PPA', action='append', help='Add ppa belonging to PPA to the vm\'s sources.list.')
         group.add_option('--lang', metavar='LANG', default=self.get_locale(), help='Set the locale to LANG [default: %default]')
+        group.add_option('--set-utc', action='store_true', help='Set the clock to UTC')
         self.vm.register_setting_group(group)
 
         group = self.vm.setting_group('Settings for the initial user')
@@ -70,6 +71,8 @@
         group.add_option('--name', default='Ubuntu', help='Full name of initial user [default: %default]')
         group.add_option('--pass', default='ubuntu', help='Password of initial user [default: %default]')
         group.add_option('--rootpass', help='Initial root password (WARNING: this has strong security implications).')
+        group.add_option('--id', help='Initial UID and GID values.')
+        group.add_option('--lock' action='store_true', help='Lock the initial user [default %default]')
         self.vm.register_setting_group(group)
 
         group = self.vm.setting_group('Other options')

=== modified file 'VMBuilder/plugins/ubuntu/hardy.py'
--- VMBuilder/plugins/ubuntu/hardy.py	2008-11-06 15:36:18 +0000
+++ VMBuilder/plugins/ubuntu/hardy.py	2009-05-13 17:13:26 +0000
@@ -22,6 +22,20 @@
 
 class Hardy(Gutsy):
     virtio_net = True
+    ec2_kernel_info = { 'i386' : 'aki-6e709707', 'amd64' : 'aki-6f709706' }
+    ec2_ramdisk_info = { 'i386' : 'ari-6c709705', 'amd64' : 'ari-61709708' }
+
+    def install_ec2(self):
+        if not self.vm.ec2:
+            return False
+
+        if self.vm.addpkg:
+            self.vm.addpkg = []
+
+        self.vm.addpkg += ['libc6-xen', 'ibc6-i686-']
+        self.install_from_template('/etc/event.d/xvc0', 'upstart', { 'console' : 'xvc0' })
+        self.install_from_template('/etc/ld.so.conf.d/libc6-xen.conf', 'xen-ld-so-conf')
+        self.run_in_target('update-rc.d', '-f', 'hwclockfirst.sh', 'remove')
 
     def xen_kernel_path(self):
         return '/boot/vmlinuz-2.6.24-19-xen'

=== modified file 'VMBuilder/plugins/ubuntu/intrepid.py'
--- VMBuilder/plugins/ubuntu/intrepid.py	2009-02-17 14:39:16 +0000
+++ VMBuilder/plugins/ubuntu/intrepid.py	2009-05-13 12:37:45 +0000
@@ -29,6 +29,17 @@
                        'lpia'  : ['lpia', 'lpiacompat'] }
     default_flavour = { 'i386' : 'virtual', 'amd64' : 'virtual', 'lpia' : 'lpia' }
     xen_kernel_flavour = 'virtual'
+    ec2_kernel_info = { 'i386' : 'aki-714daa18', 'amd64' : 'aki-4f4daa26' }
+    ec2_ramdisk_info = { 'i386': 'ari-7e4daa17', 'amd64' : 'ari-4c4daa25' }
+
+    def install_ec2(self):
+        if not self.vm.ec2:
+            return False
+
+        if not self.vm.addpkg:
+            self.vm.addpkg = []
+
+        self.vm.addpkg += ['policykit', '^server']
 
     def mangle_grub_menu_lst(self):
         bootdev = disk.bootpart(self.vm.disks)

=== modified file 'VMBuilder/plugins/ubuntu/jaunty.py'
--- VMBuilder/plugins/ubuntu/jaunty.py	2009-05-07 21:00:20 +0000
+++ VMBuilder/plugins/ubuntu/jaunty.py	2009-05-13 12:39:10 +0000
@@ -25,6 +25,17 @@
 
 class Jaunty(Intrepid):
     xen_kernel_flavour = 'server'
+    ec2_kernel_info = { 'i386' : 'aki-c553b4ac', 'amd64' : 'aki-d653b4bf' }
+    ec2_ramdisk_info = { 'i386' : 'ari-c253b4ab', 'amd64' : 'ari-d753b4be' }
+
+    def install_ec2(self):
+        if not self.vm.ec2:
+            return False
+
+        if self.vm.addpkg:
+            self.vm.addpkg = []
+
+        self.vm.addpkg = ['^server']
 
     def mangle_grub_menu_lst(self):
         bootdev = disk.bootpart(self.vm.disks)