Merge lp:~abentley/workspace-runner/s3-artifacts into lp:workspace-runner

Proposed by Aaron Bentley on 2015-06-30
Status: Merged
Merged at revision: 21
Proposed branch: lp:~abentley/workspace-runner/s3-artifacts
Merge into: lp:workspace-runner
Prerequisite: lp:~abentley/workspace-runner/s3-script
Diff against target: 352 lines (+168/-36)
4 files modified
upload.yaml (+6/-0)
workspace_runner/__init__.py (+64/-6)
workspace_runner/tests/__init__.py (+95/-27)
workspace_runner/upload_artifacts.py (+3/-3)
To merge this branch: bzr merge lp:~abentley/workspace-runner/s3-artifacts
Reviewer Review Type Date Requested Status
Curtis Hovey (community) code 2015-06-30 Approve on 2015-06-30
Review via email: mp+263384@code.launchpad.net

Commit message

Upload artifacts to s3 from workspace runner.

Description of the change

This branch applies the upload_artifacts script.

It updates workspace_run to accept artifact_prefix and --s3-config. It uses the access-key and secret key from the s3config file, plus the artifact prefix, plus the 'bucket' and 'artifacts' config values to generate a config file. That file is then used to remotely run upload_artifacts.

To post a comment you must log in.
37. By Aaron Bentley on 2015-06-30

Merged trunk into s3-artifacts.

Curtis Hovey (sinzui) wrote :

Thank you. I have a question/suggestion inline.

review: Approve (code)
Aaron Bentley (abentley) wrote :

AFAICT, the main advantage of SafeConfigParser over RawConfigParser is that SafeConfigParser supports interpolation. I didn't think interpolation was useful for this case so, I went with RawConfigParser.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added file 'upload.yaml'
2--- upload.yaml 1970-01-01 00:00:00 +0000
3+++ upload.yaml 2015-06-30 18:41:09 +0000
4@@ -0,0 +1,6 @@
5+command: [ls > foo]
6+install:
7+ docs: [README]
8+artifacts:
9+ ls_output: [foo]
10+bucket: ws-runner-test
11
12=== modified file 'workspace_runner/__init__.py'
13--- workspace_runner/__init__.py 2015-06-25 17:05:49 +0000
14+++ workspace_runner/__init__.py 2015-06-30 18:41:09 +0000
15@@ -1,15 +1,20 @@
16 from argparse import ArgumentParser
17+from ConfigParser import RawConfigParser
18 from contextlib import contextmanager
19 from itertools import chain
20+import json
21 import logging
22 from pipes import quote
23 import os
24+from shutil import rmtree
25 import subprocess
26 import sys
27+from tempfile import mkdtemp
28 from textwrap import dedent
29
30-from yaml import safe_load
31-
32+from yaml import (
33+ safe_load,
34+ )
35 __metaclass__ = type
36
37
38@@ -18,9 +23,13 @@
39 parser = ArgumentParser()
40 parser.add_argument('config', help='Config file to use.')
41 parser.add_argument('host', help='Machine to run the command on.')
42+ parser.add_argument('artifact_prefix', nargs='?',
43+ help='Prefix to use storing artifacts.')
44 parser.add_argument('--private-key', '-i', help='Private SSH key to use.')
45 parser.add_argument('--verbose', '-v', help='Verbose output.',
46 action='store_true')
47+ parser.add_argument('--s3-config',
48+ help='s3cmd config file for credentials.')
49 return parser.parse_args(argv)
50
51
52@@ -185,7 +194,49 @@
53 primitives.destroy()
54
55
56-def workspace_run(argv=None, primitives_factory=SSHPrimitives):
57+@contextmanager
58+def temp_dir():
59+ temp_dir = mkdtemp()
60+ try:
61+ yield temp_dir
62+ finally:
63+ rmtree(temp_dir)
64+
65+
66+@contextmanager
67+def temp_config(config):
68+ with temp_dir() as config_dir:
69+ config_filename = os.path.join(config_dir, 'upload.json')
70+ with open(config_filename, 'w') as config_file:
71+ json.dump(config, config_file)
72+ config_file.flush()
73+ yield config_filename
74+
75+
76+def run_from_config(primitives, config, s3_config, artifact_prefix, output):
77+ """Install the files and run the command."""
78+ for target, sources in config['install'].items():
79+ primitives.install(sources, target)
80+ if 'artifacts' in config:
81+ import upload_artifacts
82+ sources = [upload_artifacts.__file__.replace('.pyc', '.py')]
83+ upload_config = {
84+ 'bucket': config['bucket'],
85+ 'files': config['artifacts'],
86+ 'prefix': artifact_prefix,
87+ }
88+ upload_config.update(s3_config)
89+ with temp_config(upload_config) as upload_config_filename:
90+ sources.append(upload_config_filename)
91+ primitives.install(sources, '.wsr')
92+ primitives.run(config['command'], output)
93+ if 'artifacts' in config:
94+ primitives.run(['python', '.wsr/upload_artifacts.py',
95+ '.wsr/upload.json', primitives.workspace], output)
96+
97+
98+def workspace_run(argv=None, primitives_factory=SSHPrimitives,
99+ output=sys.stdout):
100 """Run an operation in a workspace."""
101 args = parse_args(argv)
102 if args.verbose:
103@@ -195,8 +246,15 @@
104 logging.basicConfig(level=level)
105 with open(args.config) as config_file:
106 config = safe_load(config_file)
107+ s3_config = None
108+ if args.s3_config is not None:
109+ config_parser = RawConfigParser()
110+ config_parser.read(args.s3_config)
111+ s3_config = {
112+ 'access_key': config_parser.get('default', 'access_key'),
113+ 'secret_key': config_parser.get('default', 'secret_key'),
114+ }
115 with workspace_context(args.host, args.private_key,
116 primitives_factory) as runner:
117- for target, sources in config['install'].items():
118- runner.install(sources, target)
119- runner.run(config['command'], sys.stdout)
120+ run_from_config(runner, config, s3_config, args.artifact_prefix,
121+ output)
122
123=== modified file 'workspace_runner/tests/__init__.py'
124--- workspace_runner/tests/__init__.py 2015-06-25 17:05:49 +0000
125+++ workspace_runner/tests/__init__.py 2015-06-30 18:41:09 +0000
126@@ -1,5 +1,6 @@
127 from argparse import Namespace
128 from contextlib import contextmanager
129+import json
130 import os
131 import logging
132 from mock import patch
133@@ -9,6 +10,7 @@
134 rmtree,
135 )
136 from StringIO import StringIO
137+import sys
138 from tempfile import (
139 mkdtemp,
140 NamedTemporaryFile,
141@@ -16,7 +18,6 @@
142 from textwrap import dedent
143 from unittest import TestCase
144 import subprocess
145-
146 from yaml import safe_dump
147
148 from workspace_runner import (
149@@ -25,26 +26,21 @@
150 retry_ssh,
151 SSHConnection,
152 SSHPrimitives,
153+ temp_config,
154+ temp_dir,
155+ run_from_config,
156 workspace_context,
157 workspace_run,
158 )
159
160
161-@contextmanager
162-def temp_dir():
163- temp_dir = mkdtemp()
164- try:
165- yield temp_dir
166- finally:
167- rmtree(temp_dir)
168-
169-
170 class TestParseArgs(TestCase):
171
172 def test_minimal(self):
173 args = parse_args(['foo', 'bar'])
174- self.assertEqual(args, Namespace(config='foo', host='bar',
175- private_key=None, verbose=False))
176+ self.assertEqual(args, Namespace(
177+ config='foo', host='bar', private_key=None, verbose=False,
178+ s3_config=None, artifact_prefix=None))
179
180 def test_private_key(self):
181 args = parse_args(['foo', 'bar', '--private-key', 'key'])
182@@ -58,6 +54,10 @@
183 args = parse_args(['foo', 'bar', '-v'])
184 self.assertEqual(args.verbose, True)
185
186+ def test_s3_config(self):
187+ args = parse_args(['foo', 'bar', '--s3-config', 'foobar'])
188+ self.assertEqual(args.s3_config, 'foobar')
189+
190
191 class FakePrimitives(Primitives):
192
193@@ -364,26 +364,75 @@
194 self.assertFalse(os.path.exists(primitives.workspace))
195
196
197+class TestRunFromConfig(TestCase):
198+
199+ def test_minimal(self):
200+ config = {
201+ 'command': ['run', 'this'],
202+ 'install': {},
203+ }
204+ with workspace_context('foo', None, FakePrimitives) as primitives:
205+ run_from_config(primitives, config, None, None, StringIO())
206+ self.assertEqual(primitives.run_calls, [['run', 'this']])
207+ self.assertEqual(primitives.walk_results,
208+ [(primitives.workspace, [], [])])
209+
210+ def test_s3_upload(self):
211+ config = {
212+ 'command': ['run', 'this'],
213+ 'install': {},
214+ 'artifacts': {'foo': ['bar']},
215+ 'bucket': 'bucket1',
216+ }
217+ s3_config = {'access_key': 'access1', 'secret_key': 'secret1'}
218+ with workspace_context('foo', None, FakePrimitives) as primitives:
219+ run_from_config(primitives, config, s3_config, 'prefix/midfix',
220+ StringIO())
221+ upload_json_path = os.path.join(primitives.workspace, '.wsr',
222+ 'upload.json')
223+ with open(upload_json_path) as remote_config_file:
224+ remote_config = json.load(remote_config_file)
225+ self.assertEqual(remote_config, {
226+ 'access_key': 'access1',
227+ 'secret_key': 'secret1',
228+ 'bucket': 'bucket1',
229+ 'files': {'foo': ['bar']},
230+ 'prefix': 'prefix/midfix',
231+ })
232+ self.assertEqual(primitives.run_calls, [
233+ ['run', 'this'],
234+ ['python', '.wsr/upload_artifacts.py', '.wsr/upload.json',
235+ primitives.workspace],
236+ ])
237+ self.assertEqual(primitives.walk_results, [
238+ (primitives.workspace, ['.wsr'], []),
239+ (primitives.workspace + '/.wsr', [], [
240+ 'upload.json', 'upload_artifacts.py'])
241+ ])
242+
243+
244 class TestWorkspaceRun(TestCase):
245
246 @contextmanager
247 def config_file(self):
248- with NamedTemporaryFile() as config_file:
249- safe_dump({
250- 'command': ['run', 'this'],
251- 'install': {},
252- }, config_file)
253- yield config_file
254+ config = {
255+ 'command': ['run', 'this'],
256+ 'install': {},
257+ }
258+ with temp_config(config) as config_file_name:
259+ yield config_file_name
260
261- def run_primitives(self, args):
262+ def run_primitives(self, args, output=None):
263 fp_factory = FakePrimitivesFactory()
264- workspace_run(args, fp_factory)
265+ if output is None:
266+ output = StringIO()
267+ workspace_run(args, fp_factory, output=output)
268 return fp_factory.last_instance
269
270 def test_minimal(self):
271- with self.config_file() as config_file:
272+ with self.config_file() as config_file_name:
273 with patch('logging.root.handlers', []):
274- primitives = self.run_primitives([config_file.name, 'bar'])
275+ primitives = self.run_primitives([config_file_name, 'bar'])
276 self.assertEqual(logging.getLogger().getEffectiveLevel(),
277 logging.WARNING)
278 self.assertEqual(primitives.run_calls, [['run', 'this']])
279@@ -392,9 +441,9 @@
280 [(primitives.workspace, [], [])])
281
282 def test_private_key(self):
283- with self.config_file() as config_file:
284+ with self.config_file() as config_file_name:
285 primitives = self.run_primitives(
286- [config_file.name, 'bar', '-i', 'qux'])
287+ [config_file_name, 'bar', '-i', 'qux'])
288 self.assertEqual(primitives.ssh_connection.private_key, 'qux')
289
290 def test_install(self):
291@@ -407,7 +456,7 @@
292 'install': {'bin-dir': [install_file.name]},
293 }, config_file)
294 workspace_run([config_file.name, 'bar'],
295- fp_factory)
296+ fp_factory, StringIO())
297 primitives = fp_factory.last_instance
298 bin_path = os.path.join(primitives.workspace, 'bin-dir')
299 install_base = os.path.basename(install_file.name)
300@@ -417,8 +466,27 @@
301 ])
302
303 def test_verbose(self):
304- with self.config_file() as config_file:
305+ with self.config_file() as config_file_name:
306 with patch('logging.root.handlers', []):
307- self.run_primitives([config_file.name, 'bar', '-v'])
308+ self.run_primitives([config_file_name, 'bar', '-v'])
309 self.assertEqual(logging.getLogger().getEffectiveLevel(),
310 logging.INFO)
311+
312+ def test_s3config(self):
313+ config = {'command': ['run', 'this'], 'install': {}}
314+ credentials = {'access_key': 'foobar', 'secret_key': 'barfoo'}
315+ with NamedTemporaryFile() as s3_config_file:
316+ s3_config_file.write(dedent("""\
317+ [default]
318+ access_key = foobar
319+ secret_key = barfoo
320+ """))
321+ s3_config_file.flush()
322+ with self.config_file() as config_file_name:
323+ with patch('workspace_runner.run_from_config',
324+ autospec=True) as rfc_mock:
325+ argv = [config_file_name, 'bar',
326+ '--s3-config', s3_config_file.name]
327+ primitives = self.run_primitives(argv, output=sys.stdout)
328+ rfc_mock.assert_called_once_with(
329+ primitives, config, credentials, None, sys.stdout)
330
331=== modified file 'workspace_runner/upload_artifacts.py'
332--- workspace_runner/upload_artifacts.py 2015-06-30 18:41:09 +0000
333+++ workspace_runner/upload_artifacts.py 2015-06-30 18:41:09 +0000
334@@ -11,8 +11,8 @@
335
336 def parse_args(argv=None):
337 parser = ArgumentParser()
338- parser.add_argument('artifacts_file')
339- parser.add_argument('root')
340+ parser.add_argument('artifacts_file', help='Configuration file.')
341+ parser.add_argument('root', help='The root directory to upload from.')
342 return parser.parse_args(argv)
343
344
345@@ -40,7 +40,7 @@
346 args = parse_args(argv)
347 # Use JSON rather than YAML because a program will emit it and no external
348 # libs required.
349- with file(args.artifacts_file) as settings_file:
350+ with open(args.artifacts_file) as settings_file:
351 settings = json.load(settings_file)
352 upload_artifacts(args.root, settings)
353

Subscribers

People subscribed via source and target branches