Merge lp:~sseman/workspace-runner/download-logs into lp:workspace-runner

Proposed by Seman on 2015-10-08
Status: Merged
Merged at revision: 25
Proposed branch: lp:~sseman/workspace-runner/download-logs
Merge into: lp:workspace-runner
Diff against target: 126 lines (+62/-0)
3 files modified
examples/download-dir.yaml (+5/-0)
workspace_runner/__init__.py (+17/-0)
workspace_runner/tests/__init__.py (+40/-0)
To merge this branch: bzr merge lp:~sseman/workspace-runner/download-logs
Reviewer Review Type Date Requested Status
Aaron Bentley (community) 2015-10-08 Approve on 2015-10-09
Review via email: mp+273788@code.launchpad.net

Description of the change

This branch adds a support for copying a log (or any other) directory from remote to local. When "logs" is added to the config file, it will copy the stated directory from remote machine to local:

install:
  doc: [README]
command: ["cool", "command"]
logs:
    remote_logs: "/workspace/logs"

In the above example, it will copy the remote directory "remote_logs" to local "/workspace/logs" directory. The "remote_logs" path is relative to the temp directory created by the Workspace Runner.

To post a comment you must log in.
Aaron Bentley (abentley) wrote :

I can live with the functionality, but I don't think "logs" is the right word, because
1. it's downloading directories containing logs, not just logs.
2. it can download arbitrary directories, not just those containing logs.

I suggest "download-dir".

Please add an example yaml file.

review: Needs Fixing
26. By Seman on 2015-10-08

Changed logs to download-dir and added example.

Seman (sseman) wrote :

Created an example and renamed "logs" to 'download-dir".

Aaron Bentley (abentley) wrote :

Please rename test_logs to test_download_dir. Otherwise, please land.

review: Approve
27. By Seman on 2015-10-09

Changed the name of test_logs to test_download_dir.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added directory 'examples'
2=== added file 'examples/download-dir.yaml'
3--- examples/download-dir.yaml 1970-01-01 00:00:00 +0000
4+++ examples/download-dir.yaml 2015-10-09 18:32:36 +0000
5@@ -0,0 +1,5 @@
6+install:
7+ docs: [README]
8+command: [mkdir remote-log && wc docs/README > remote-log/wordcount.log]
9+download-dir:
10+ remote-log: "/tmp/logs"
11\ No newline at end of file
12
13=== renamed file 'example.yaml' => 'examples/example.yaml'
14=== modified file 'workspace_runner/__init__.py'
15--- workspace_runner/__init__.py 2015-07-17 13:58:50 +0000
16+++ workspace_runner/__init__.py 2015-10-09 18:32:36 +0000
17@@ -125,6 +125,15 @@
18 logging.info('Running: {}'.format(' '.join(command)))
19 subprocess.check_call(command)
20
21+ @retry_ssh((1, 255))
22+ def scp_dir_from_remote(self, source, dest_path):
23+ """Copy dir from the specified path on the remote host."""
24+ source_loc = '{}:{}/'.format(self.host, quote(source))
25+ command = (['scp'] + ['-r'] + self.get_ssh_options() + [source_loc] +
26+ [dest_path])
27+ logging.info('Running: {}'.format(' '.join(command)))
28+ subprocess.check_call(command)
29+
30
31 class SSHPrimitives(Primitives):
32
33@@ -182,6 +191,11 @@
34 self.mkdir(dest_path)
35 self.ssh_connection.scp_to_remote(sources, dest_path)
36
37+ def copy_dir(self, source, target):
38+ """Copy dir from remote to local."""
39+ source_path = os.path.join(self.workspace, source)
40+ self.ssh_connection.scp_dir_from_remote(source_path, target)
41+
42
43 @contextmanager
44 def workspace_context(host, private_key, primitives_factory):
45@@ -248,6 +262,9 @@
46 primitives.install(sources, '.wsr')
47 primitives.run(['python', '.wsr/upload_artifacts.py',
48 '.wsr/upload.json', primitives.workspace], output)
49+ if 'download-dir' in config:
50+ for source, target in config['download-dir'].items():
51+ primitives.copy_dir(source, target)
52
53
54 def workspace_run(argv=None, primitives_factory=SSHPrimitives,
55
56=== modified file 'workspace_runner/tests/__init__.py'
57--- workspace_runner/tests/__init__.py 2015-07-17 13:58:50 +0000
58+++ workspace_runner/tests/__init__.py 2015-10-09 18:32:36 +0000
59@@ -7,6 +7,7 @@
60 from pipes import quote
61 from shutil import (
62 copyfile,
63+ copytree,
64 rmtree,
65 )
66 from StringIO import StringIO
67@@ -85,6 +86,11 @@
68 target_path = os.path.join(ws_target, os.path.basename(source))
69 copyfile(source, target_path)
70
71+ def copy_dir(self, source, target):
72+ ws_target = os.path.join(self.workspace, target)
73+ target_path = os.path.join(ws_target, os.path.basename(source))
74+ copytree(source, target_path)
75+
76
77 class TestPrimitives:
78
79@@ -224,6 +230,15 @@
80 with self.assertRaises(subprocess.CalledProcessError):
81 connection.call_with_input([], '')
82
83+ def test_scp_dir_from_remote(self):
84+ connection = SSHConnection('host@example.com', 'key')
85+ with patch('subprocess.check_call', autospec=True) as pc_mock:
86+ connection.scp_dir_from_remote('source', 'dest')
87+ pc_mock.assert_called_once_with(
88+ ['scp', '-r', '-oStrictHostKeyChecking=no',
89+ '-oUserKnownHostsFile=/dev/null', '-i', 'key',
90+ 'host@example.com:source/', 'dest'])
91+
92
93 class TestSSHPrimitives(TestPrimitives, TestCase):
94
95@@ -511,6 +526,31 @@
96 (bin_path, [], [install_base]),
97 ])
98
99+ def test_download_dir(self):
100+ fp_factory = FakePrimitivesFactory()
101+ with temp_dir() as log_dir:
102+ sub_log_dir = os.path.join(log_dir, "machine-0")
103+ os.makedirs(sub_log_dir)
104+ log_file = os.path.join(sub_log_dir, 'machine-0.log')
105+ open(log_file, 'w').close()
106+ with temp_dir() as dest_log_dir:
107+ with NamedTemporaryFile() as config_file:
108+ safe_dump({
109+ 'command': ['run', 'this'],
110+ 'install': {'bin-dir': [log_file]},
111+ 'download-dir': {log_dir: dest_log_dir},
112+ }, config_file)
113+ workspace_run([config_file.name, 'bar'],
114+ fp_factory, StringIO())
115+ log_dir_base = os.path.basename(log_dir)
116+ log_dir_path = os.path.join(dest_log_dir, log_dir_base)
117+ machine_log_path = os.path.join(log_dir_path, 'machine-0')
118+ self.assertEqual(
119+ list(os.walk(dest_log_dir)),
120+ [(dest_log_dir, [log_dir_base], []),
121+ (log_dir_path, ['machine-0'], []),
122+ (machine_log_path, [], ['machine-0.log'])])
123+
124 def test_verbose(self):
125 with self.config_file() as config_file_name:
126 with patch('logging.root.handlers', []):

Subscribers

People subscribed via source and target branches