Merge lp:~cprov/adt-cloud-worker/swift-uploads into lp:adt-cloud-worker

[Celso Providelo]

Uploading adt results to swift as adt-<req_id>/results.tgz.

[Evan (ev)]

Looks good. Just a couple of side questions.

review: Approve

[lp:~cprov/adt-cloud-worker/swift-uploads]

9. By Celso Providelo on 2015-03-03

Worker uploads results.tgz.tmp to swift and rely on result checker to rename it if it's valid.

[Celso Providelo (cprov)]

Evan,

Thanks for the review, comments addressed.

=== modified file 'adt-cloud-worker.py'
--- adt-cloud-worker.py	2015-03-03 16:27:28 +0000
+++ adt-cloud-worker.py	2015-03-03 19:03:39 +0000
@@ -31,6 +31,8 @@
 import os
 import shutil
 import subprocess
+from swiftclient import client
+import tarfile
 import tempfile
 
 logger = get_logger(__name__)
@@ -61,31 +63,55 @@
         """
         logger.info('Got: {}'.format(body))
         # TODO: body validation
-        message.ack()
         body['worker'] = self.name
         # Run requested tests safely.
         try:
             # Create and temporary directory for storing adt results.
             result_dir = tempfile.mkdtemp(
                 prefix='adt-{}-{}'.format(self.name, body['request_id']))
+            # Run `adt-run`.
             adt_kwargs = body.copy()
             adt_kwargs['result_dir'] = result_dir
             body['exit_code'] = self.run_adt(**adt_kwargs)
-            # TODO: tar/gzip the output directory.
-            # TODO: upload to swift instance at 'swift_container'
+            # Compress the results directory in a temporary file.
+            _unused, tarball_path = tempfile.mkstemp()
+            with tarfile.open(tarball_path, "w:gz") as tar:
+                for entry in os.listdir(result_dir):
+                    tar.add(os.path.join(result_dir, entry), arcname=entry)
+            # Upload the results tarball to swift with a temporary name
+            # 'adt-<req_id>/results.tgz.tmp', result checker will rename
+            # if after verification.
+            swift_client = client.Connection(
+                authurl=self.config.get('nova', 'os_auth_url'),
+                user=self.config.get('nova', 'os_username'),
+                key=self.config.get('nova', 'os_password'),
+                os_options={
+                    'tenant_name': self.config.get('nova', 'os_tenant_name'),
+                    'region_name': self.config.get('nova', 'os_region_name'),
+                }, auth_version=(
+                    '/v1.' in self.config.get('nova','os_auth_url') and '1.0'
+                    or '2.0'))
+            container_name = 'adt-{}'.format(body['request_id'])
+            swift_client.put_container(container_name)
+            with open(tarball_path) as fd:
+                swift_client.put_object(
+                    container_name, obj='results.tgz.tmp', contents=fd,
+                    content_type='application/x-compressed')
         except Exception as e:
             # Unexpected failures ... 
             logger.error(e, exc_info=True)
             body['exit_code'] = '100'
         finally:
-            # Drop the result directory and post results
-            shutil.rmtree(result_dir)
+            # The post results
             # TODO: send error logging to result-checker in the message
-            # TODO: resilience agains rabbit hiccups, otherwise the request
-            # will be lost.
             queue = self.connection.SimpleQueue('adt.results')
             queue.put(body)
             queue.close()
+            # Drop the result directory and result tarball.
+            shutil.rmtree(result_dir)
+            os.remove(tarball_path)
+            message.ack()
+            logger.info('Done!')
 
     def run_adt(self, **kwargs):
         """Run adt-run according to the given request parameters.