Merge lp:~bcsaller/charmtester/ssh-redux into lp:charmtester

[Benjamin Saller]

more verbose output, better ssh interactions

[Benjamin Saller (bcsaller)]

*** Submitted:

more verbose output, better ssh interactions

=== modified file 'setup.py'
--- setup.py	2014-01-18 08:52:42 +0000
+++ setup.py	2014-01-28 19:43:54 +0000
@@ -18,12 +18,13 @@
     license="MIT",
     keywords="testing substates juju",
     description=README,
-    entry_points = {
+    entry_points={
         "console_scripts": [
             'substrate_dispatch = substrateDispatcher.dispatcher:main',
-            'jssh = substrateDispatcher.jlxc:main',
+            'jssh = substrateDispatcher.jlxc:ssh',
             'jhere = substrateDispatcher.jlxc:here',
-            'jscp = substrateDispatcher.jlxc:scp'
+            'jscp = substrateDispatcher.jlxc:scp',
+            'jstart = substrateDispatcher.jlxc:start'
         ]
     }
 )

=== modified file 'substrateDispatcher/jlxc.py'
--- substrateDispatcher/jlxc.py	2014-01-27 21:22:38 +0000
+++ substrateDispatcher/jlxc.py	2014-01-28 19:43:54 +0000
@@ -9,8 +9,6 @@
 import tempfile
 import time
 
-from multiprocessing import Process
-
 from .identity import Identity
 
 log = default_log = logging.getLogger("jlxc")
@@ -21,36 +19,35 @@
         return self[key]
 
 
-def concurrent(fn):
-    """Simple concurrency decorator"""
-    def run(*args, **kwargs):
-        c = Process(target=fn, args=args, kwargs=kwargs)
-        c.start()
-        c.join()
-        return c
-    return run
-
-
 def run(cmd, **kwargs):
     if isinstance(cmd, str):
         cmd = shlex.split(cmd)
     pipe = subprocess.PIPE
     fatal = kwargs.pop('fatal', False)
+    inline = kwargs.pop('inline', False)
+    on_failure = kwargs.pop('on_failure', None)
+    stdout = kwargs.pop('stdout', inline and sys.stdout or pipe)
+
     cmdstr = ' '.join(cmd)
     log.debug("Exec Command: {}".format(cmdstr))
+
     process = subprocess.Popen(
-        cmd, stdout=kwargs.pop('stdout', pipe),
+        cmd,
+        stdout=stdout,
         stderr=kwargs.pop('stderr', pipe),
         close_fds=kwargs.pop('close_fds', True), **kwargs)
     stdout, stderr = process.communicate()
     if process.returncode != 0:
         log.debug('Error invoking cmd: {} -> {}'.format(
             cmdstr, process.returncode))
-        if fatal:
+        if fatal or on_failure:
             exception = subprocess.CalledProcessError(
                 process.returncode, cmdstr)
             exception.output = ''.join(filter(None, [stdout, stderr]))
-            raise exception
+            if on_failure:
+                on_failure(cmd, process, exception)
+            if fatal:
+                raise exception
     return O(returncode=process.returncode,
              stdout=stdout, stderr=stderr)
 
@@ -165,7 +162,7 @@
         # they will skip so it makes sense to do this in the base image.
         with LXCContainer(series_container) as c:
             c.set_identity(self.identity)
-            c.start(True)
+            c.start()
             c.run_template('customize_container',
                            fatal=True)
             c.stop()
@@ -234,11 +231,24 @@
     def start(self, wait=False):
         if self.running:
             return
-        params = ["sudo", "lxc-start", '-d']
+        log_path = os.path.join(self.container_root,
+                                "%s-container.log" % self.name)
+
+        def dump_log(cmd, process, exc):
+            with open(log_path, 'r') as fp:
+                output = fp.read()
+                exc_output = getattr(exc, 'output')
+                if exc_output:
+                    output += exc_output
+                log.error("Command failure: %s\n%s",
+                          cmd, output, exc_info=exc)
+
+        params = ["sudo", "lxc-start", '-d',
+                  '-o', log_path]
         if self.container_root:
             params.extend(["-P", self.container_root])
         params.extend(["--name", self.name])
-        run(params, fatal=True)
+        run(params, fatal=True, on_failure=dump_log)
 
         if wait:
             self.wait_for_running(wait)
@@ -343,7 +353,7 @@
                 return result
             return self.ssh(dest, **kwargs)
 
-    def run_template(self, template, inline=False, **kwargs):
+    def run_template(self, template, **kwargs):
         if not os.path.exists(template):
             template = os.path.join('templates', template)
         if not os.path.exists(template):
@@ -352,11 +362,9 @@
         script = open(template, 'r').read()
         script = script.format(**kwargs)
         logging.debug("Executing HERE document::\n{}".format(script))
-        return self.here(script,
-                         stdout=sys.stdout if inline else None,
-                         fatal=fatal)
+        return self.here(script, fatal=fatal)
 
-    def wait_for(self, interval, timeout, *callbacks):
+    def wait_for(self, timeout, interval, *callbacks):
         """
         Repeatedly try callbacks until all return True
 
@@ -376,12 +384,13 @@
             if passes is True:
                 break
             current = time.time()
-            if current - start >= timeout:
-                raise OSError("Timeout exceeded in waitFor")
+            if current - start >= timeout or \
+                    current + interval > timeout:
+                raise OSError("Timeout exceeded in wait_for")
             time.sleep(interval)
 
-    def wait_for_running(self, interval=5, timeout=30):
-        self.wait_for(interval, timeout, valid_ip, running_ssh)
+    def wait_for_running(self, timeout=30, interval=2):
+        self.wait_for(timeout, interval, valid_ip, running_ssh)
 
 
 # wait_for callbacks
@@ -398,7 +407,7 @@
         s = socket.socket()
         # check ssh port
         addr = (ip, 22)
-        logging.info('Waiting on ssh: %s', addr)
+        logging.debug('Waiting on ssh: %s', addr)
         s.connect(addr)
         s.close()
         return True
@@ -407,34 +416,40 @@
         return False
 
 
+def _normalize_container(value):
+    if value.endswith('/'):
+        value = value[:-1]
+    name, path = (os.path.basename(value),
+                  os.path.dirname(value))
+    return name, path
+
+
+def _start_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-l', '--log-level',
+                        dest="log_level", default=logging.INFO)
+    parser.add_argument('--timeout', type=int, default=35)
+    return parser
+
+
 def here():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-l', '--log-level',
-                        dest="log_level", default=logging.INFO)
+    parser = _start_parser()
     parser.add_argument('container')
     parser.add_argument('script', type=file)
     options = parser.parse_args()
-
     logging.basicConfig(level=options.log_level)
-    if options.container.endswith('/'):
-        options.container = options.container[:-1]
+    name, path = _normalize_container(options.container)
+    identity = Identity.find()
 
-    with LXCManager(container_root=os.path.dirname(options.container),
-                    identity=Identity.find(),
-                    cleanup=False,
-                    destroy=False) as m:
-        with m.container(os.path.basename(options.container)) as c:
-            if not c.running:
-                c.start()
-            c.here(options.script,
-                   user=options.user,
-                   identity=options.ssh_identity)
+    with LXCContainer(name, path) as c:
+        c.set_identity(identity)
+        if not c.running:
+            c.start(options.timeout)
+        c.here(options.script)
 
 
 def scp():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-l', '--log-level', dest="log_level",
-                        default=logging.INFO)
+    parser = _start_parser()
     parser.add_argument('-r', '--recursive', action="store_true")
     parser.add_argument('container')
     parser.add_argument('source')
@@ -442,46 +457,58 @@
     options = parser.parse_args()
 
     logging.basicConfig(level=options.log_level)
-    if options.container.endswith('/'):
-        options.container = options.container[:-1]
-
-    with LXCManager(container_root=os.path.dirname(options.container),
-                    identity=Identity.find(),
-                    cleanup=False,
-                    destroy=False) as m:
-        with m.container(os.path.basename(options.container)) as c:
-            if not c.running:
-                c.start()
-            c.scp(options.source, options.dest,
-                  recursive=options.recursive,
-                  user=options.user, identity=options.ssh_identity)
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-l', '--log-level',
-                        dest="log_level", default=logging.INFO)
+    name, path = _normalize_container(options.container)
+    identity = Identity.find()
+
+    with LXCContainer(name, path) as c:
+        c.set_identity(identity)
+        if not c.running:
+            c.start(options.timeout)
+        c.scp(options.source, options.dest,
+              recursive=options.recursive)
+
+
+def ssh():
+    parser = _start_parser()
     parser.add_argument('container')
     parser.add_argument('cmd', nargs="+")
     options = parser.parse_args()
 
     logging.basicConfig(level=options.log_level)
-    if options.container.endswith('/'):
-        options.container = options.container[:-1]
-
-    with LXCManager(container_root=os.path.dirname(options.container),
-                    identity=Identity.find(),
-                    cleanup=False,
-                    destroy=False) as m:
-        with m.container(os.path.basename(options.container)) as c:
-            if not c.running:
-                c.start()
-            result = c.ssh(options.cmd, user=options.user,
-                           identity=options.ssh_identity)
-            if result.stdout:
-                print(result.stdout)
-            if result.stderr:
-                print(result.stderr)
-
-if __name__ == '__main__':
-    main()
+    name, path = _normalize_container(options.container)
+    identity = Identity.find()
+
+    with LXCContainer(name, path) as c:
+        c.set_identity(identity)
+        if not c.running:
+            c.start(options.timeout)
+        result = c.ssh(options.cmd)
+
+        if result.stdout:
+            print(result.stdout)
+        if result.stderr:
+            print(result.stderr)
+
+
+def start():
+    parser = _start_parser()
+    parser.add_argument('container')
+    options = parser.parse_args()
+
+    logging.basicConfig(level=options.log_level)
+    name, path = _normalize_container(options.container)
+    identity = Identity.find()
+
+    c = LXCContainer(name, path)
+    c.set_identity(identity)
+    if not c.running:
+        c.start(options.timeout)
+    print('Container will be left running, to stop:\n'
+          'sudo lxc-stop -P {} -n {}'.format(
+              c.container_root, c.name))
+    sys.stdout.flush()
+
+    ssh_args = ["ssh", "-o", "StrictHostKeyChecking=no",
+                "-o", "UserKnownHostsFile=/dev/null",
+                "ubuntu@{}".format(c.info()['ipv4'])]
+    os.execvp('ssh', ssh_args)