Merge lp:~sseman/workspace-runner/download-logs into lp:workspace-runner

[Seman]

This branch adds a support for copying a log (or any other) directory from remote to local. When "logs" is added to the config file, it will copy the stated directory from remote machine to local:

install:
  doc: [README]
command: ["cool", "command"]
logs:
    remote_logs: "/workspace/logs"

In the above example, it will copy the remote directory "remote_logs" to local "/workspace/logs" directory. The "remote_logs" path is relative to the temp directory created by the Workspace Runner.

[Aaron Bentley (abentley)]

I can live with the functionality, but I don't think "logs" is the right word, because
1. it's downloading directories containing logs, not just logs.
2. it can download arbitrary directories, not just those containing logs.

I suggest "download-dir".

Please add an example yaml file.

review: Needs Fixing

[lp:~sseman/workspace-runner/download-logs]

26. By Seman on 2015-10-08

Changed logs to download-dir and added example.

[Seman (sseman)]

Created an example and renamed "logs" to 'download-dir".

[Aaron Bentley (abentley)]

Please rename test_logs to test_download_dir. Otherwise, please land.

review: Approve

[lp:~sseman/workspace-runner/download-logs]

27. By Seman on 2015-10-09

Changed the name of test_logs to test_download_dir.

=== added directory 'examples'
=== added file 'examples/download-dir.yaml'
--- examples/download-dir.yaml	1970-01-01 00:00:00 +0000
+++ examples/download-dir.yaml	2015-10-09 18:32:36 +0000
@@ -0,0 +1,5 @@
+install:
+  docs: [README]
+command: [mkdir remote-log && wc docs/README > remote-log/wordcount.log]
+download-dir:
+  remote-log: "/tmp/logs"
\ No newline at end of file

=== renamed file 'example.yaml' => 'examples/example.yaml'
=== modified file 'workspace_runner/__init__.py'
--- workspace_runner/__init__.py	2015-07-17 13:58:50 +0000
+++ workspace_runner/__init__.py	2015-10-09 18:32:36 +0000
@@ -125,6 +125,15 @@
         logging.info('Running: {}'.format(' '.join(command)))
         subprocess.check_call(command)
 
+    @retry_ssh((1, 255))
+    def scp_dir_from_remote(self, source, dest_path):
+        """Copy dir from the specified path on the remote host."""
+        source_loc = '{}:{}/'.format(self.host, quote(source))
+        command = (['scp'] + ['-r'] + self.get_ssh_options() + [source_loc] +
+                   [dest_path])
+        logging.info('Running: {}'.format(' '.join(command)))
+        subprocess.check_call(command)
+
 
 class SSHPrimitives(Primitives):
 
@@ -182,6 +191,11 @@
         self.mkdir(dest_path)
         self.ssh_connection.scp_to_remote(sources, dest_path)
 
+    def copy_dir(self, source, target):
+        """Copy dir from remote to local."""
+        source_path = os.path.join(self.workspace, source)
+        self.ssh_connection.scp_dir_from_remote(source_path, target)
+
 
 @contextmanager
 def workspace_context(host, private_key, primitives_factory):
@@ -248,6 +262,9 @@
             primitives.install(sources, '.wsr')
         primitives.run(['python', '.wsr/upload_artifacts.py',
                         '.wsr/upload.json', primitives.workspace], output)
+    if 'download-dir' in config:
+        for source, target in config['download-dir'].items():
+            primitives.copy_dir(source, target)
 
 
 def workspace_run(argv=None, primitives_factory=SSHPrimitives,

=== modified file 'workspace_runner/tests/__init__.py'
--- workspace_runner/tests/__init__.py	2015-07-17 13:58:50 +0000
+++ workspace_runner/tests/__init__.py	2015-10-09 18:32:36 +0000
@@ -7,6 +7,7 @@
 from pipes import quote
 from shutil import (
     copyfile,
+    copytree,
     rmtree,
     )
 from StringIO import StringIO
@@ -85,6 +86,11 @@
             target_path = os.path.join(ws_target, os.path.basename(source))
             copyfile(source, target_path)
 
+    def copy_dir(self, source, target):
+        ws_target = os.path.join(self.workspace, target)
+        target_path = os.path.join(ws_target, os.path.basename(source))
+        copytree(source, target_path)
+
 
 class TestPrimitives:
 
@@ -224,6 +230,15 @@
             with self.assertRaises(subprocess.CalledProcessError):
                 connection.call_with_input([], '')
 
+    def test_scp_dir_from_remote(self):
+        connection = SSHConnection('[e-mail]', 'key')
+        with patch('subprocess.check_call', autospec=True) as pc_mock:
+            connection.scp_dir_from_remote('source', 'dest')
+        pc_mock.assert_called_once_with(
+            ['scp', '-r', '-oStrictHostKeyChecking=no',
+             '-oUserKnownHostsFile=/dev/null', '-i', 'key',
+             '[e-mail]:source/', 'dest'])
+
 
 class TestSSHPrimitives(TestPrimitives, TestCase):
 
@@ -511,6 +526,31 @@
             (bin_path, [], [install_base]),
             ])
 
+    def test_download_dir(self):
+        fp_factory = FakePrimitivesFactory()
+        with temp_dir() as log_dir:
+            sub_log_dir = os.path.join(log_dir, "machine-0")
+            os.makedirs(sub_log_dir)
+            log_file = os.path.join(sub_log_dir, 'machine-0.log')
+            open(log_file, 'w').close()
+            with temp_dir() as dest_log_dir:
+                with NamedTemporaryFile() as config_file:
+                    safe_dump({
+                        'command': ['run', 'this'],
+                        'install': {'bin-dir': [log_file]},
+                        'download-dir': {log_dir: dest_log_dir},
+                        }, config_file)
+                    workspace_run([config_file.name, 'bar'],
+                                  fp_factory, StringIO())
+                    log_dir_base = os.path.basename(log_dir)
+                    log_dir_path = os.path.join(dest_log_dir, log_dir_base)
+                    machine_log_path = os.path.join(log_dir_path, 'machine-0')
+                    self.assertEqual(
+                        list(os.walk(dest_log_dir)),
+                        [(dest_log_dir, [log_dir_base], []),
+                         (log_dir_path,  ['machine-0'], []),
+                         (machine_log_path, [], ['machine-0.log'])])
+
     def test_verbose(self):
         with self.config_file() as config_file_name:
             with patch('logging.root.handlers', []):