Merge lp:~bcsaller/pyjuju/lxc-clone-support into lp:pyjuju

[Benjamin Saller]

lxc-clone support

This branch includes support for lxc-clone which is later used to speed the deployment of containers in the scope of a bootstrap.

[Kapil Thangavelu (hazmat)]

i believe this merge is out of date, the relevant functionality has already been merged, setting it to rejected to clean out the lp review queue.

Unmerged revisions

323. By Benjamin Saller on 2011-09-15

fix wait condition in cleanup, how did that ever work

322. By Benjamin Saller on 2011-09-14

lxc-clone support in library

=== modified file 'ensemble/lib/lxc/__init__.py'
--- ensemble/lib/lxc/__init__.py	2011-09-08 23:53:09 +0000
+++ ensemble/lib/lxc/__init__.py	2011-09-15 00:54:25 +0000
@@ -18,13 +18,14 @@
 
 
 def _cmd(args):
-    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    p = subprocess.Popen(
+        args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     stdout_data, _ = p.communicate()
     r = p.returncode
     if  r != 0:
         # read the stdout/err streams and show the user
         print >>sys.stderr, stdout_data
-        raise LXCError(stdout_data)
+        raise LXCError("%s: %s" % (" ".join(args), stdout_data))
     return (r, stdout_data)
 
 
@@ -42,11 +43,21 @@
     return _cmd(args)
 
 
-def _lxc_start(container_name, console_log=None):
+def _lxc_clone(orig_container, container_name):
+    return _cmd(["sudo", "lxc-clone", "-o", orig_container, "-n",
+                 container_name])
+
+
+def _lxc_start(container_name, debug_log=None, console_log=None):
     args = ["sudo", "lxc-start", "--daemon", "-n", container_name]
     if console_log:
         args.append("-c")
         args.append(console_log)
+    if debug_log:
+        args.append("-l")
+        args.append("DEBUG")
+        args.append("-o")
+        args.append(debug_log)
     return _cmd(args)
 
 
@@ -76,7 +87,7 @@
     return deferToThread(wait, container_name)
 
 
-def _customize_container(customize_script, container_root):
+def _customize_container(customize_script, container_root, *args):
     if not os.path.isdir(container_root):
         raise LXCError("Expect container root directory: %s" %
                        container_root)
@@ -88,9 +99,11 @@
     os.close(fd)
     os.chmod(in_path, 0755)
 
-    args = ["sudo", "chroot", container_root,
-            os.path.join("/tmp", os.path.basename(in_path))]
-    return _cmd(args)
+    cmd = ["sudo", "chroot", container_root,
+           os.path.join("/tmp", os.path.basename(in_path))]
+    if args:
+        cmd.extend(args)
+    return _cmd(cmd)
 
 
 def validate_path(pathname):
@@ -124,8 +137,11 @@
 
 class LXCContainer(object):
     def __init__(self,
-                 container_name, configuration=None, network_name="virbr0",
-                 customize_script=None):
+                 container_name, configuration=None,
+                 network_name="virbr0",
+                 template_name=None,
+                 customize_script=None,
+                 debug_log=None, console_log=None):
         """Create an LXCContainer
 
         :param container_name: should be unique within the system
@@ -138,9 +154,15 @@
 
         :param customize_script: script used inside container to make
         it ensemble ready
+
+        :param template_name: name of an existing lxc template to use
+        a a basis for clone operations (rather than create). This will
+        still customize the resulting container.
         """
 
         self.container_name = container_name
+        self.debug_log = debug_log
+        self.console_log = console_log
 
         if customize_script is None:
             customize_script = os.path.join(DATA_PATH,
@@ -148,8 +170,13 @@
         self.customize_script = customize_script
         validate_path(self.customize_script)
 
+        if not configuration:
+            configuration = {}
+        configuration["ensemble_container_name"] = container_name
+
         self.configuration = configuration
         self.network_name = network_name
+        self.template_name = template_name
         self.running = False
 
     @property
@@ -175,20 +202,31 @@
 
     @inlineCallbacks
     def _customize_container(self):
+        if not self.configuration:
+            return
+
         config_dir = self._p("etc/ensemble")
         if not os.path.exists(config_dir):
             _cmd(["sudo", "mkdir", config_dir])
 
-        if not self.configuration:
-            return
-
         _, fn = tempfile.mkstemp()
+
         with open(fn, "w") as fp:
+            target = self._p("etc/ensemble/ensemble.conf")
+
+            if self.template_name:
+                # there may exist a config in the client already we
+                # copy its values to the new file and then append any
+                # additional settings even though its possible the
+                # file will have redundant entries they will override
+                # the templates due to order of operations
+                if os.path.exists(target):
+                    fp.write(open(target, "r").read())
+
             for name, value in self.configuration.items():
                 fp.write("%s=%s\n" % (name.upper(), pipes.quote(value)))
 
-        _cmd(["sudo", "mv", fn,
-              self._p("etc/ensemble/ensemble.conf")])
+        _cmd(["sudo", "mv", fn, target])
 
         yield deferToThread(
             _customize_container, self.customize_script, self.rootfs)
@@ -205,18 +243,40 @@
     def create(self):
         # open the template file and create a new temp processed
         # version
-        lxc_config = self._make_lxc_config(self.network_name)
-        yield deferToThread(_lxc_create, self.container_name,
-                            config_file=lxc_config)
+        if self.template_name is not None:
+            # we are cloning from a template
+            yield deferToThread(
+                _lxc_clone, self.template_name, self.container_name)
+        else:
+            lxc_config = self._make_lxc_config(self.network_name)
+            yield deferToThread(_lxc_create, self.container_name,
+                                config_file=lxc_config)
+            os.unlink(lxc_config)
+
         yield self._customize_container()
-        os.unlink(lxc_config)
+
+    def clone(self, container_name):
+        if self.configuration:
+            config = dict(self.configuration)
+        else:
+            config = None
+
+        return LXCContainer(container_name,
+                            template_name=self.container_name,
+                            configuration=config,
+                            debug_log=self.debug_log,
+                            console_log=self.console_log,
+                            customize_script=self.customize_script,
+                            network_name=self.network_name)
 
     @inlineCallbacks
     def run(self):
         if not os.path.exists(self.rootfs):
             yield self.create()
 
-        yield deferToThread(_lxc_start, self.container_name)
+        yield deferToThread(
+            _lxc_start, self.container_name,
+            debug_log=self.debug_log, console_log=self.console_log)
         self.running = yield _lxc_wait(self.container_name, "RUNNING")
 
     @inlineCallbacks

=== modified file 'ensemble/lib/lxc/tests/test_lxc.py'
--- ensemble/lib/lxc/tests/test_lxc.py	2011-09-08 23:53:09 +0000
+++ ensemble/lib/lxc/tests/test_lxc.py	2011-09-15 00:54:25 +0000
@@ -49,6 +49,7 @@
     def cleanContainer(self, container_name):
         if os.path.exists("/var/lib/lxc/%s" % container_name):
             _lxc_stop(container_name)
+            _lxc_wait(container_name, "STOPPED")
             _lxc_destroy(container_name)
 
     def test_lxc_create(self):
@@ -96,11 +97,6 @@
         # verify we have a path into the container
         self.assertTrue(os.path.exists(c.rootfs))
 
-        # expect a processed config file above the rootfs
-        # -- this requires root to read
-        #network = open(os.path.join(c.rootfs, "..", "config"), "r").read()
-        #self.assertIn("lxc.network.link=virbr0", network)
-
         # check that some of the customize modifications were made
         self.assertTrue(os.path.exists(os.path.join(
                     c.rootfs, "etc", "ensemble")))
@@ -145,7 +141,60 @@
         yield _lxc_wait(DEFAULT_CONTAINER, "STOPPED")
         _lxc_destroy(DEFAULT_CONTAINER)
 
-
-
-
+    @inlineCallbacks
+    def test_container_clone(self):
+        d =  dict(ensemble_zookeeper=get_test_zookeeper_address(),
+                  ensemble_origin="distro")
+        master_container = LXCContainer(DEFAULT_CONTAINER,
+                                        configuration=d)
+        yield master_container.create()
+
+        # Clone a child container from the template
+        child_name = DEFAULT_CONTAINER + "_child"
+        c = master_container.clone(child_name)
+
+        expected = dict(d)
+        expected["ensemble_container_name"] =  DEFAULT_CONTAINER + "_child"
+        self.assertEqual(c.configuration, expected)
+        self.assertFalse(c.running)
+        yield c.run()
+        self.assertTrue(c.running)
+
+        output = _lxc_ls()
+        self.assertIn(DEFAULT_CONTAINER, output)
+
+        # verify we have a path into the container
+        self.assertTrue(os.path.exists(c.rootfs))
+
+        # expect a processed config file above the rootfs
+        # -- this requires root to read
+        #network = open(os.path.join(c.rootfs, "..", "config"), "r").read()
+        #self.assertIn("lxc.network.link=virbr0", network)
+
+        # check that some of the customize modifications were made
+        self.assertTrue(os.path.exists(os.path.join(
+                    c.rootfs, "etc", "ensemble")))
+        self.assertTrue(os.path.exists(os.path.join(
+                    c.rootfs, "usr", "bin", "ensemble")))
+
+        # verify that we have shell variables in rootfs/etc/ensemble.conf
+        config = open(os.path.join(c.rootfs, "etc", "ensemble", "ensemble.conf"), "r").read()
+        self.assertIn("ENSEMBLE_ZOOKEEPER=%s" % get_test_zookeeper_address(), config)
+
+        # verify that we are in containers
+        containers = yield get_containers(None)
+        self.assertEqual(containers[child_name], True)
+
+        # tear it down
+        yield c.destroy()
+        self.assertFalse(c.running)
+
+        containers = yield get_containers(None)
+        self.assertNotIn(child_name, containers)
+
+        # and its gone
+        output = _lxc_ls()
+        self.assertNotIn(child_name, output)
+
+        yield master_container.destroy()