Merge lp:~abentley/workspace-runner/s3-artifacts into lp:workspace-runner

Proposed by Aaron Bentley on 2015-06-30
Status: Merged
Merged at revision: 21
Proposed branch: lp:~abentley/workspace-runner/s3-artifacts
Merge into: lp:workspace-runner
Prerequisite: lp:~abentley/workspace-runner/s3-script
Diff against target: 352 lines (+168/-36)
4 files modified
upload.yaml (+6/-0)
workspace_runner/__init__.py (+64/-6)
workspace_runner/tests/__init__.py (+95/-27)
workspace_runner/upload_artifacts.py (+3/-3)
To merge this branch: bzr merge lp:~abentley/workspace-runner/s3-artifacts
Reviewer Review Type Date Requested Status
Curtis Hovey (community) code 2015-06-30 Approve on 2015-06-30
Review via email: mp+263384@code.launchpad.net

Commit message

Upload artifacts to s3 from workspace runner.

Description of the change

This branch applies the upload_artifacts script.

It updates workspace_run to accept artifact_prefix and --s3-config. It uses the access-key and secret key from the s3config file, plus the artifact prefix, plus the 'bucket' and 'artifacts' config values to generate a config file. That file is then used to remotely run upload_artifacts.

To post a comment you must log in.
37. By Aaron Bentley on 2015-06-30

Merged trunk into s3-artifacts.

Curtis Hovey (sinzui) wrote :

Thank you. I have a question/suggestion inline.

review: Approve (code)
Aaron Bentley (abentley) wrote :

AFAICT, the main advantage of SafeConfigParser over RawConfigParser is that SafeConfigParser supports interpolation. I didn't think interpolation was useful for this case so, I went with RawConfigParser.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== added file 'upload.yaml'
--- upload.yaml 1970-01-01 00:00:00 +0000
+++ upload.yaml 2015-06-30 18:41:09 +0000
@@ -0,0 +1,6 @@
1command: [ls > foo]
2install:
3 docs: [README]
4artifacts:
5 ls_output: [foo]
6bucket: ws-runner-test
07
=== modified file 'workspace_runner/__init__.py'
--- workspace_runner/__init__.py 2015-06-25 17:05:49 +0000
+++ workspace_runner/__init__.py 2015-06-30 18:41:09 +0000
@@ -1,15 +1,20 @@
1from argparse import ArgumentParser1from argparse import ArgumentParser
2from ConfigParser import RawConfigParser
2from contextlib import contextmanager3from contextlib import contextmanager
3from itertools import chain4from itertools import chain
5import json
4import logging6import logging
5from pipes import quote7from pipes import quote
6import os8import os
9from shutil import rmtree
7import subprocess10import subprocess
8import sys11import sys
12from tempfile import mkdtemp
9from textwrap import dedent13from textwrap import dedent
1014
11from yaml import safe_load15from yaml import (
1216 safe_load,
17 )
13__metaclass__ = type18__metaclass__ = type
1419
1520
@@ -18,9 +23,13 @@
18 parser = ArgumentParser()23 parser = ArgumentParser()
19 parser.add_argument('config', help='Config file to use.')24 parser.add_argument('config', help='Config file to use.')
20 parser.add_argument('host', help='Machine to run the command on.')25 parser.add_argument('host', help='Machine to run the command on.')
26 parser.add_argument('artifact_prefix', nargs='?',
27 help='Prefix to use storing artifacts.')
21 parser.add_argument('--private-key', '-i', help='Private SSH key to use.')28 parser.add_argument('--private-key', '-i', help='Private SSH key to use.')
22 parser.add_argument('--verbose', '-v', help='Verbose output.',29 parser.add_argument('--verbose', '-v', help='Verbose output.',
23 action='store_true')30 action='store_true')
31 parser.add_argument('--s3-config',
32 help='s3cmd config file for credentials.')
24 return parser.parse_args(argv)33 return parser.parse_args(argv)
2534
2635
@@ -185,7 +194,49 @@
185 primitives.destroy()194 primitives.destroy()
186195
187196
188def workspace_run(argv=None, primitives_factory=SSHPrimitives):197@contextmanager
198def temp_dir():
199 temp_dir = mkdtemp()
200 try:
201 yield temp_dir
202 finally:
203 rmtree(temp_dir)
204
205
206@contextmanager
207def temp_config(config):
208 with temp_dir() as config_dir:
209 config_filename = os.path.join(config_dir, 'upload.json')
210 with open(config_filename, 'w') as config_file:
211 json.dump(config, config_file)
212 config_file.flush()
213 yield config_filename
214
215
216def run_from_config(primitives, config, s3_config, artifact_prefix, output):
217 """Install the files and run the command."""
218 for target, sources in config['install'].items():
219 primitives.install(sources, target)
220 if 'artifacts' in config:
221 import upload_artifacts
222 sources = [upload_artifacts.__file__.replace('.pyc', '.py')]
223 upload_config = {
224 'bucket': config['bucket'],
225 'files': config['artifacts'],
226 'prefix': artifact_prefix,
227 }
228 upload_config.update(s3_config)
229 with temp_config(upload_config) as upload_config_filename:
230 sources.append(upload_config_filename)
231 primitives.install(sources, '.wsr')
232 primitives.run(config['command'], output)
233 if 'artifacts' in config:
234 primitives.run(['python', '.wsr/upload_artifacts.py',
235 '.wsr/upload.json', primitives.workspace], output)
236
237
238def workspace_run(argv=None, primitives_factory=SSHPrimitives,
239 output=sys.stdout):
189 """Run an operation in a workspace."""240 """Run an operation in a workspace."""
190 args = parse_args(argv)241 args = parse_args(argv)
191 if args.verbose:242 if args.verbose:
@@ -195,8 +246,15 @@
195 logging.basicConfig(level=level)246 logging.basicConfig(level=level)
196 with open(args.config) as config_file:247 with open(args.config) as config_file:
197 config = safe_load(config_file)248 config = safe_load(config_file)
249 s3_config = None
250 if args.s3_config is not None:
251 config_parser = RawConfigParser()
252 config_parser.read(args.s3_config)
253 s3_config = {
254 'access_key': config_parser.get('default', 'access_key'),
255 'secret_key': config_parser.get('default', 'secret_key'),
256 }
198 with workspace_context(args.host, args.private_key,257 with workspace_context(args.host, args.private_key,
199 primitives_factory) as runner:258 primitives_factory) as runner:
200 for target, sources in config['install'].items():259 run_from_config(runner, config, s3_config, args.artifact_prefix,
201 runner.install(sources, target)260 output)
202 runner.run(config['command'], sys.stdout)
203261
=== modified file 'workspace_runner/tests/__init__.py'
--- workspace_runner/tests/__init__.py 2015-06-25 17:05:49 +0000
+++ workspace_runner/tests/__init__.py 2015-06-30 18:41:09 +0000
@@ -1,5 +1,6 @@
1from argparse import Namespace1from argparse import Namespace
2from contextlib import contextmanager2from contextlib import contextmanager
3import json
3import os4import os
4import logging5import logging
5from mock import patch6from mock import patch
@@ -9,6 +10,7 @@
9 rmtree,10 rmtree,
10 )11 )
11from StringIO import StringIO12from StringIO import StringIO
13import sys
12from tempfile import (14from tempfile import (
13 mkdtemp,15 mkdtemp,
14 NamedTemporaryFile,16 NamedTemporaryFile,
@@ -16,7 +18,6 @@
16from textwrap import dedent18from textwrap import dedent
17from unittest import TestCase19from unittest import TestCase
18import subprocess20import subprocess
19
20from yaml import safe_dump21from yaml import safe_dump
2122
22from workspace_runner import (23from workspace_runner import (
@@ -25,26 +26,21 @@
25 retry_ssh,26 retry_ssh,
26 SSHConnection,27 SSHConnection,
27 SSHPrimitives,28 SSHPrimitives,
29 temp_config,
30 temp_dir,
31 run_from_config,
28 workspace_context,32 workspace_context,
29 workspace_run,33 workspace_run,
30 )34 )
3135
3236
33@contextmanager
34def temp_dir():
35 temp_dir = mkdtemp()
36 try:
37 yield temp_dir
38 finally:
39 rmtree(temp_dir)
40
41
42class TestParseArgs(TestCase):37class TestParseArgs(TestCase):
4338
44 def test_minimal(self):39 def test_minimal(self):
45 args = parse_args(['foo', 'bar'])40 args = parse_args(['foo', 'bar'])
46 self.assertEqual(args, Namespace(config='foo', host='bar',41 self.assertEqual(args, Namespace(
47 private_key=None, verbose=False))42 config='foo', host='bar', private_key=None, verbose=False,
43 s3_config=None, artifact_prefix=None))
4844
49 def test_private_key(self):45 def test_private_key(self):
50 args = parse_args(['foo', 'bar', '--private-key', 'key'])46 args = parse_args(['foo', 'bar', '--private-key', 'key'])
@@ -58,6 +54,10 @@
58 args = parse_args(['foo', 'bar', '-v'])54 args = parse_args(['foo', 'bar', '-v'])
59 self.assertEqual(args.verbose, True)55 self.assertEqual(args.verbose, True)
6056
57 def test_s3_config(self):
58 args = parse_args(['foo', 'bar', '--s3-config', 'foobar'])
59 self.assertEqual(args.s3_config, 'foobar')
60
6161
62class FakePrimitives(Primitives):62class FakePrimitives(Primitives):
6363
@@ -364,26 +364,75 @@
364 self.assertFalse(os.path.exists(primitives.workspace))364 self.assertFalse(os.path.exists(primitives.workspace))
365365
366366
367class TestRunFromConfig(TestCase):
368
369 def test_minimal(self):
370 config = {
371 'command': ['run', 'this'],
372 'install': {},
373 }
374 with workspace_context('foo', None, FakePrimitives) as primitives:
375 run_from_config(primitives, config, None, None, StringIO())
376 self.assertEqual(primitives.run_calls, [['run', 'this']])
377 self.assertEqual(primitives.walk_results,
378 [(primitives.workspace, [], [])])
379
380 def test_s3_upload(self):
381 config = {
382 'command': ['run', 'this'],
383 'install': {},
384 'artifacts': {'foo': ['bar']},
385 'bucket': 'bucket1',
386 }
387 s3_config = {'access_key': 'access1', 'secret_key': 'secret1'}
388 with workspace_context('foo', None, FakePrimitives) as primitives:
389 run_from_config(primitives, config, s3_config, 'prefix/midfix',
390 StringIO())
391 upload_json_path = os.path.join(primitives.workspace, '.wsr',
392 'upload.json')
393 with open(upload_json_path) as remote_config_file:
394 remote_config = json.load(remote_config_file)
395 self.assertEqual(remote_config, {
396 'access_key': 'access1',
397 'secret_key': 'secret1',
398 'bucket': 'bucket1',
399 'files': {'foo': ['bar']},
400 'prefix': 'prefix/midfix',
401 })
402 self.assertEqual(primitives.run_calls, [
403 ['run', 'this'],
404 ['python', '.wsr/upload_artifacts.py', '.wsr/upload.json',
405 primitives.workspace],
406 ])
407 self.assertEqual(primitives.walk_results, [
408 (primitives.workspace, ['.wsr'], []),
409 (primitives.workspace + '/.wsr', [], [
410 'upload.json', 'upload_artifacts.py'])
411 ])
412
413
367class TestWorkspaceRun(TestCase):414class TestWorkspaceRun(TestCase):
368415
369 @contextmanager416 @contextmanager
370 def config_file(self):417 def config_file(self):
371 with NamedTemporaryFile() as config_file:418 config = {
372 safe_dump({419 'command': ['run', 'this'],
373 'command': ['run', 'this'],420 'install': {},
374 'install': {},421 }
375 }, config_file)422 with temp_config(config) as config_file_name:
376 yield config_file423 yield config_file_name
377424
378 def run_primitives(self, args):425 def run_primitives(self, args, output=None):
379 fp_factory = FakePrimitivesFactory()426 fp_factory = FakePrimitivesFactory()
380 workspace_run(args, fp_factory)427 if output is None:
428 output = StringIO()
429 workspace_run(args, fp_factory, output=output)
381 return fp_factory.last_instance430 return fp_factory.last_instance
382431
383 def test_minimal(self):432 def test_minimal(self):
384 with self.config_file() as config_file:433 with self.config_file() as config_file_name:
385 with patch('logging.root.handlers', []):434 with patch('logging.root.handlers', []):
386 primitives = self.run_primitives([config_file.name, 'bar'])435 primitives = self.run_primitives([config_file_name, 'bar'])
387 self.assertEqual(logging.getLogger().getEffectiveLevel(),436 self.assertEqual(logging.getLogger().getEffectiveLevel(),
388 logging.WARNING)437 logging.WARNING)
389 self.assertEqual(primitives.run_calls, [['run', 'this']])438 self.assertEqual(primitives.run_calls, [['run', 'this']])
@@ -392,9 +441,9 @@
392 [(primitives.workspace, [], [])])441 [(primitives.workspace, [], [])])
393442
394 def test_private_key(self):443 def test_private_key(self):
395 with self.config_file() as config_file:444 with self.config_file() as config_file_name:
396 primitives = self.run_primitives(445 primitives = self.run_primitives(
397 [config_file.name, 'bar', '-i', 'qux'])446 [config_file_name, 'bar', '-i', 'qux'])
398 self.assertEqual(primitives.ssh_connection.private_key, 'qux')447 self.assertEqual(primitives.ssh_connection.private_key, 'qux')
399448
400 def test_install(self):449 def test_install(self):
@@ -407,7 +456,7 @@
407 'install': {'bin-dir': [install_file.name]},456 'install': {'bin-dir': [install_file.name]},
408 }, config_file)457 }, config_file)
409 workspace_run([config_file.name, 'bar'],458 workspace_run([config_file.name, 'bar'],
410 fp_factory)459 fp_factory, StringIO())
411 primitives = fp_factory.last_instance460 primitives = fp_factory.last_instance
412 bin_path = os.path.join(primitives.workspace, 'bin-dir')461 bin_path = os.path.join(primitives.workspace, 'bin-dir')
413 install_base = os.path.basename(install_file.name)462 install_base = os.path.basename(install_file.name)
@@ -417,8 +466,27 @@
417 ])466 ])
418467
419 def test_verbose(self):468 def test_verbose(self):
420 with self.config_file() as config_file:469 with self.config_file() as config_file_name:
421 with patch('logging.root.handlers', []):470 with patch('logging.root.handlers', []):
422 self.run_primitives([config_file.name, 'bar', '-v'])471 self.run_primitives([config_file_name, 'bar', '-v'])
423 self.assertEqual(logging.getLogger().getEffectiveLevel(),472 self.assertEqual(logging.getLogger().getEffectiveLevel(),
424 logging.INFO)473 logging.INFO)
474
475 def test_s3config(self):
476 config = {'command': ['run', 'this'], 'install': {}}
477 credentials = {'access_key': 'foobar', 'secret_key': 'barfoo'}
478 with NamedTemporaryFile() as s3_config_file:
479 s3_config_file.write(dedent("""\
480 [default]
481 access_key = foobar
482 secret_key = barfoo
483 """))
484 s3_config_file.flush()
485 with self.config_file() as config_file_name:
486 with patch('workspace_runner.run_from_config',
487 autospec=True) as rfc_mock:
488 argv = [config_file_name, 'bar',
489 '--s3-config', s3_config_file.name]
490 primitives = self.run_primitives(argv, output=sys.stdout)
491 rfc_mock.assert_called_once_with(
492 primitives, config, credentials, None, sys.stdout)
425493
=== modified file 'workspace_runner/upload_artifacts.py'
--- workspace_runner/upload_artifacts.py 2015-06-30 18:41:09 +0000
+++ workspace_runner/upload_artifacts.py 2015-06-30 18:41:09 +0000
@@ -11,8 +11,8 @@
1111
12def parse_args(argv=None):12def parse_args(argv=None):
13 parser = ArgumentParser()13 parser = ArgumentParser()
14 parser.add_argument('artifacts_file')14 parser.add_argument('artifacts_file', help='Configuration file.')
15 parser.add_argument('root')15 parser.add_argument('root', help='The root directory to upload from.')
16 return parser.parse_args(argv)16 return parser.parse_args(argv)
1717
1818
@@ -40,7 +40,7 @@
40 args = parse_args(argv)40 args = parse_args(argv)
41 # Use JSON rather than YAML because a program will emit it and no external41 # Use JSON rather than YAML because a program will emit it and no external
42 # libs required.42 # libs required.
43 with file(args.artifacts_file) as settings_file:43 with open(args.artifacts_file) as settings_file:
44 settings = json.load(settings_file)44 settings = json.load(settings_file)
45 upload_artifacts(args.root, settings)45 upload_artifacts(args.root, settings)
4646

Subscribers

People subscribed via source and target branches