Merge lp:~abentley/workspace-runner/s3-script into lp:workspace-runner

[Aaron Bentley]

Provide a script that uploads to s3.

This branch provides the upload_artifacts script, which will typically be run remotely, but is tested locally.

It uses a JSON file for data. (JSON so that it doesn't require external dependencies.)

It doesn't currently pay attention to MIME types, but I plan to address that in future branch.

In a the follow-on branch, the workspace runner is taught to dump this JSON, upload it and the script to the remote machine, and run the script.

[John George (jog)]

Would you please add help= to add_arguments() in upload_artifacts.py, other than that it looks good.

review: Approve

[Curtis Hovey (sinzui)]

Thank you. I have a suggestion inline

review: Approve (code)

=== added file 'workspace_runner/tests/test_upload_artifacts.py'
--- workspace_runner/tests/test_upload_artifacts.py	1970-01-01 00:00:00 +0000
+++ workspace_runner/tests/test_upload_artifacts.py	2015-06-30 16:28:18 +0000
@@ -0,0 +1,123 @@
+from argparse import Namespace
+import json
+import os
+from tempfile import NamedTemporaryFile
+from unittest import TestCase
+
+from mock import patch
+
+from workspace_runner.tests import temp_dir
+from workspace_runner.upload_artifacts import (
+    main,
+    parse_args,
+    resolve_rules,
+    upload_artifacts,
+    )
+
+
+class TestParseArgs(TestCase):
+
+    def test_minimal(self):
+        result = parse_args(['artifacts1', 'root1'])
+        self.assertEqual(result,
+                         Namespace(artifacts_file='artifacts1', root='root1'))
+
+
+class TestResolveRules(TestCase):
+
+    def test_resolve_rules(self):
+        settings = {'files': {
+            'text-files': ['*.txt', '*.rst'],
+            'foo-files': ['foo.*'],
+            'subdir-files': ['subdir/*'],
+            }}
+        filenames = ['foo.txt', 'bar.gif', 'foo.gif', 'subdir/baz.txt',
+                     'qux.rst']
+        with temp_dir() as files_dir:
+            os.mkdir(os.path.join(files_dir, 'subdir'))
+            for filename in filenames:
+                open(os.path.join(files_dir, filename), 'w').close()
+            result = resolve_rules(files_dir, settings['files'])
+            expected = {
+                (os.path.join(files_dir, 'foo.txt'), 'text-files/foo.txt'),
+                (os.path.join(files_dir, 'qux.rst'), 'text-files/qux.rst'),
+                (os.path.join(files_dir, 'foo.txt'), 'foo-files/foo.txt'),
+                (os.path.join(files_dir, 'foo.gif'), 'foo-files/foo.gif'),
+                (os.path.join(files_dir, 'subdir/baz.txt'),
+                 'subdir-files/baz.txt'),
+                }
+            self.assertEqual(result, expected)
+
+
+class FakeS3Connection:
+
+    def __init__(self, access_key, secret_key):
+        self.access_key = access_key
+        self.secret_key = secret_key
+        self.buckets = {}
+
+    def get_bucket(self, bucket_name):
+        return self.buckets.setdefault(bucket_name, FakeBucket())
+
+
+class FakeBucket:
+
+    def __init__(self):
+        self.keys = []
+
+    def new_key(self, path):
+        self.keys.append(FakeKey(path))
+        return self.keys[-1]
+
+
+class FakeKey:
+
+    def __init__(self, path):
+        self.contents = None
+        self.path = path
+
+    def set_contents_from_file(self, fp):
+        self.contents = fp.read()
+
+
+class TestUploadArtifacts(TestCase):
+
+    def test_upload_artifacts(self):
+        settings = {
+            'access_key': 'baz',
+            'secret_key': 'qux',
+            'bucket': 'my-pale-bucket',
+            'files': {'text-files': ['*.txt', '*.rst']},
+            'prefix': 'prefix1',
+            }
+
+        connections = []
+
+        def fake_factory(*args, **kwargs):
+            connections.append(FakeS3Connection(*args, **kwargs))
+            return connections[-1]
+
+        with temp_dir() as files_dir:
+            with open(os.path.join(files_dir, 'foo.txt'), 'w') as foo_file:
+                foo_file.write('foo file')
+            upload_artifacts(files_dir, settings, s3_factory=fake_factory)
+        self.assertEqual(len(connections), 1)
+        connection = connections[0]
+        self.assertEqual(connection.access_key, 'baz')
+        self.assertEqual(connection.secret_key, 'qux')
+        bucket = connection.buckets['my-pale-bucket']
+        key = bucket.keys[0]
+        self.assertEqual(key.path, 'prefix1/text-files/foo.txt')
+        self.assertEqual(key.contents, 'foo file')
+
+
+class TestMain(TestCase):
+
+    def test_minimal(self):
+        with NamedTemporaryFile() as settings_file:
+            json.dump({'foo': 'bar'}, settings_file)
+            settings_file.flush()
+            with patch('workspace_runner.upload_artifacts.upload_artifacts',
+                       autospec=True) as ua_mock:
+                main([settings_file.name, 'root1'])
+        ua_mock.assert_called_once_with('root1', {'foo': 'bar'})
 
=== added file 'workspace_runner/upload_artifacts.py'
--- workspace_runner/upload_artifacts.py	1970-01-01 00:00:00 +0000
+++ workspace_runner/upload_artifacts.py	2015-06-30 16:28:18 +0000
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# NOTE: This script is typically run on the REMOTE machine.
+from argparse import ArgumentParser
+from glob import glob
+import json
+import os
+import sys
+
+from boto.s3.connection import S3Connection
+
+
+def parse_args(argv=None):
+    parser = ArgumentParser()
+    parser.add_argument('artifacts_file')
+    parser.add_argument('root')
+    return parser.parse_args(argv)
+
+
+def resolve_rules(root, rules):
+    result = set()
+    for destination, globs in rules.items():
+        for file_glob in globs:
+            glob_path = os.path.join(root, file_glob)
+            for filename in glob(glob_path):
+                dest_url = '/'.join([destination, os.path.basename(filename)])
+                result.add((filename, dest_url))
+    return result
+
+
+def upload_artifacts(root, settings, s3_factory=S3Connection):
+    connection = s3_factory(settings['access_key'], settings['secret_key'])
+    bucket = connection.get_bucket(settings['bucket'])
+    for source_name, dest in resolve_rules(root, settings['files']):
+        key = bucket.new_key('/'.join((settings['prefix'], dest)))
+        with open(source_name) as source:
+            key.set_contents_from_file(source)
+
+
+def main(argv=None):
+    args = parse_args(argv)
+    # Use JSON rather than YAML because a program will emit it and no external
+    # libs required.
+    with file(args.artifacts_file) as settings_file:
+        settings = json.load(settings_file)
+    upload_artifacts(args.root, settings)
+
+
+if __name__ == '__main__':
+    sys.exit(main())