Merge lp:~fwereade/pyjuju/late-command-change into lp:pyjuju

[William Reade]

Equivalent to lp:~andreserl/juju/cloud-init-orchestra-lp850260, but with unit tests.

[Andres Rodriguez (andreserl)]

I have tested this with lp:juju and it works as expected. Merging the changes into orchestra in a few moments. Please go ahead and merge this otherwise we would end-up with a released orchestra not working with juju.

Thanks!

[Andres Rodriguez (andreserl)]

Fixes into orchestra have been committed. Waiting on juju side to be able to release into Oneiric!

[lp:~fwereade/pyjuju/late-command-change]

350. By William Reade on 2011-09-16

merge trunk

351. By William Reade on 2011-09-16

don't depend on precise yaml output; compare loaded data instead

[Gustavo Niemeyer (niemeyer)]

This looks great, thanks guys!

review: Approve

=== modified file 'juju/providers/orchestra/launch.py'
--- juju/providers/orchestra/launch.py	2011-09-15 18:50:23 +0000
+++ juju/providers/orchestra/launch.py	2011-09-16 14:25:15 +0000
@@ -1,6 +1,4 @@
 from base64 import b64encode
-from cStringIO import StringIO
-from gzip import GzipFile
 
 from twisted.internet.defer import inlineCallbacks, returnValue
 
@@ -8,35 +6,6 @@
 
 from .machine import machine_from_dict
 
-_LATE_COMMAND_TEMPLATE = """
-seed_d=/var/lib/cloud/seed/nocloud-net
-mkdir -p "$seed_d"
-cat > "$seed_d"/meta-data <<"EOF"
-instance-id: %s
-EOF
-
-cat > "$seed_d"/user-data <<"EOF"
-%s
-EOF
-"""
-
-_KSMETA_LATE_COMMAND_TEMPLATE = (
-    "in-target sh -c '"
-        "f=$1; "
-        "shift; "
-        "echo $0 | base64 --decode | gunzip > $f "
-        "&& chmod u+x $f "
-        "&& $f $*"
-    "' %s /root/late-command")
-
-
-def _base64_gzip(content):
-    gzipped = StringIO()
-    gzip_file = GzipFile(fileobj=gzipped, mode="wb", compresslevel=9)
-    gzip_file.write(content)
-    gzip_file.close()
-    return b64encode(gzipped.getvalue())
-
 
 def _ks_meta(**kwargs):
     return " ".join('%s="%s"' % item for item in kwargs.items())
@@ -67,17 +36,10 @@
         cloud_init = self._create_cloud_init(machine_id, zookeepers)
         cloud_init.set_provider_type("orchestra")
         cloud_init.set_instance_id_accessor(instance_id)
-        ks_meta = self._build_ks_meta(
-            cloud_init.render(), machine_id, instance_id)
+        ks_meta = _ks_meta(MACHINE_ID=machine_id,
+                           USER_DATA_BASE64=b64encode(cloud_init.render()))
 
         yield cobbler.set_on_system(instance_id, "ks_meta", ks_meta)
         yield cobbler.start_system(instance_id)
         (info,) = yield cobbler.describe_systems(instance_id)
         returnValue([machine_from_dict(info)])
-
-    def _build_ks_meta(self, user_data, machine_id, instance_id):
-        late_command = _LATE_COMMAND_TEMPLATE % (instance_id, user_data)
-        encoded = _base64_gzip(late_command)
-        juju_late_command = _KSMETA_LATE_COMMAND_TEMPLATE % encoded
-        return _ks_meta(
-            MACHINE_ID=machine_id, JUJU_LATE_COMMAND=juju_late_command)

=== modified file 'juju/providers/orchestra/tests/common.py'
--- juju/providers/orchestra/tests/common.py	2011-09-15 18:50:23 +0000
+++ juju/providers/orchestra/tests/common.py	2011-09-16 14:25:15 +0000
@@ -1,10 +1,8 @@
 from base64 import b64decode
-from cStringIO import StringIO
-from gzip import GzipFile
 import os
 import re
 from xmlrpclib import Fault
-from yaml import dump
+from yaml import dump, load
 
 from twisted.internet.defer import fail, succeed
 from twisted.web.error import Error
@@ -25,15 +23,10 @@
           "authorized-keys": "this-is-a-public-key"}
 
 
-def _extract_late_command(ks_meta_leftover):
-    unpack = re.escape(
-        "in-target sh -c 'f=$1; shift; "
-        "echo $0 | base64 --decode | gunzip > $f && chmod u+x $f && $f $*' ")
-    encoded_re = re.compile(
-        '^JUJU_LATE_COMMAND="%s([^ ]*) /root/late-command"$' % unpack)
+def _extract_user_data(ks_meta_leftover):
+    encoded_re = re.compile('^USER_DATA_BASE64="([^ ]*)"$')
     encoded = encoded_re.match(ks_meta_leftover).group(1)
-    gzipped = b64decode(encoded)
-    return GzipFile(fileobj=StringIO(gzipped)).read()
+    return load(b64decode(encoded))
 
 
 class OrchestraTestMixin(object):
@@ -98,19 +91,19 @@
         self.proxy_m.callRemote("save_system", "smith", "TOKEN")
         self.mocker.result(succeed(True))
 
-    def get_verify_ks_meta(self, machine_id, late_command_filename):
+    def get_verify_ks_meta(self, machine_id, user_data_filename):
 
         def verify(ks_meta):
-            machine_id_re = re.compile(
-                '([ ]?MACHINE_ID="%s"[ ]?)' % machine_id)
-            self.assertTrue(machine_id_re.match(ks_meta))
+            seek = '([ ]?MACHINE_ID="%s"[ ]?)' % machine_id
+            machine_id_re = re.compile(seek)
+            self.assertTrue(machine_id_re.search(ks_meta))
             leftover = machine_id_re.sub("", ks_meta)
-            late_command = _extract_late_command(leftover)
+            user_data = _extract_user_data(leftover)
 
-            expect_path = os.path.join(DATA_DIR, late_command_filename)
+            expect_path = os.path.join(DATA_DIR, user_data_filename)
             with open(expect_path) as f:
-                expect_late_command = f.read()
-            self.assertEquals(late_command, expect_late_command)
+                expect_user_data = load(f.read())
+            self.assertEquals(user_data, expect_user_data)
             return True
         return verify
 

=== renamed file 'juju/providers/orchestra/tests/data/bootstrap_late_command' => 'juju/providers/orchestra/tests/data/bootstrap_user_data'
--- juju/providers/orchestra/tests/data/bootstrap_late_command	2011-09-16 00:36:31 +0000
+++ juju/providers/orchestra/tests/data/bootstrap_user_data	2011-09-16 14:25:15 +0000
@@ -1,11 +1,3 @@
-
-seed_d=/var/lib/cloud/seed/nocloud-net
-mkdir -p "$seed_d"
-cat > "$seed_d"/meta-data <<"EOF"
-instance-id: winston-uid
-EOF
-
-cat > "$seed_d"/user-data <<"EOF"
 #cloud-config
 apt-update: true
 apt-upgrade: true
@@ -23,5 +15,3 @@
   'JUJU_ZOOKEEPER=localhost:2181 python -m juju.agents.provision -n --logfile=/var/log/juju/provision-agent.log
     --pidfile=/var/run/juju/provision-agent.pid']
 ssh_authorized_keys: [this-is-a-public-key]
-
-EOF

=== renamed file 'juju/providers/orchestra/tests/data/launch_late_command' => 'juju/providers/orchestra/tests/data/launch_user_data'
--- juju/providers/orchestra/tests/data/launch_late_command	2011-09-16 00:36:31 +0000
+++ juju/providers/orchestra/tests/data/launch_user_data	2011-09-16 14:25:15 +0000
@@ -1,11 +1,3 @@
-
-seed_d=/var/lib/cloud/seed/nocloud-net
-mkdir -p "$seed_d"
-cat > "$seed_d"/meta-data <<"EOF"
-instance-id: winston-uid
-EOF
-
-cat > "$seed_d"/user-data <<"EOF"
 #cloud-config
 apt-update: true
 apt-upgrade: true
@@ -20,5 +12,3 @@
     /var/log/juju, 'JUJU_MACHINE_ID=42 JUJU_ZOOKEEPER=jennifer:2181 python -m juju.agents.machine
     -n --logfile=/var/log/juju/machine-agent.log --pidfile=/var/run/juju/machine-agent.pid']
 ssh_authorized_keys: [this-is-a-public-key]
-
-EOF

=== modified file 'juju/providers/orchestra/tests/test_bootstrap.py'
--- juju/providers/orchestra/tests/test_bootstrap.py	2011-09-15 18:50:23 +0000
+++ juju/providers/orchestra/tests/test_bootstrap.py	2011-09-16 14:25:15 +0000
@@ -67,7 +67,7 @@
         self.mock_get_systems()
         self.mock_acquire_system()
         self.mock_set_ks_meta(
-            self.get_verify_ks_meta(0, "bootstrap_late_command"), **kwargs)
+            self.get_verify_ks_meta(0, "bootstrap_user_data"), **kwargs)
         self.mocker.replay()
         d = self.get_provider().bootstrap()
         self.assertFailure(d, ProviderError)
@@ -85,7 +85,7 @@
         self.mock_get_systems()
         self.mock_acquire_system()
         self.mock_set_ks_meta(
-            self.get_verify_ks_meta(0, "bootstrap_late_command"))
+            self.get_verify_ks_meta(0, "bootstrap_user_data"))
         self.mock_start_system()
         self.mock_describe_systems(succeed([{
             "uid": "winston-uid",

=== modified file 'juju/providers/orchestra/tests/test_launch.py'
--- juju/providers/orchestra/tests/test_launch.py	2011-09-15 18:50:23 +0000
+++ juju/providers/orchestra/tests/test_launch.py	2011-09-16 14:25:15 +0000
@@ -68,7 +68,7 @@
         self.mock_get_systems()
         self.mock_acquire_system()
         self.mock_set_ks_meta(
-            self.get_verify_ks_meta(42, "launch_late_command"))
+            self.get_verify_ks_meta(42, "launch_user_data"))
         self.mock_start_system()
         self.mock_describe_systems(succeed([{
             "uid": "winston-uid",