Merge lp:~sinzui/juju-release-tools/azure-rc3 into lp:juju-release-tools

[Curtis Hovey]

switch azure_publish_tools to use azure==2.0.0rc3

The feature-slave accidentally got the old azure lib, breaking the azure HA test. We need to switch everything
to the same library to avoid this maintenance burden

This branch updates azure_publish_tools to use the new azure storage lib (azure-storage (0.31.0)). This lib is automatically installed with azure==2.0.0rc3, but is not apart of that ARM project. This lib is an evolution
of version 0.8.0. Generic blobs are avoided in this new lib.

We use the BlockBlobService now and it manages the types, so a few methods loose some args.
It works with our existing account.
It suggests it wants an SharedAccessKey, but we do not require it...we are all powerful.
The public put_blob() method was removed.
put_block() does the initial put_blob().
The block ids must be caste to BlobBlock() which appears to manage state.
The optional BlobProperties were moved into a ContentSetting object

I have put, synced, and deleted several files and this

I added test coverage for delete_files()

My next branch will deal with the pip dep changes.

[Aaron Bentley (abentley)]

This looks good. It would be nice if the credentials.yaml could be used, but I am not sure whether it contains the right values.

review: Approve

=== modified file 'azure_publish_tools.py'
--- azure_publish_tools.py	2016-02-23 14:47:09 +0000
+++ azure_publish_tools.py	2016-06-17 21:21:24 +0000
@@ -17,7 +17,12 @@
 import socket
 import sys
 
-from azure.storage import BlobService
+from azure.storage.blob import (
+    BlobBlock,
+    BlockBlobService,
+    ContentSettings,
+    Include,
+    )
 
 
 mimetypes.init()
@@ -69,11 +74,13 @@
     """Return the SyncFile info about files under the specified prefix."""
     files = []
     for blob in blob_service.list_blobs(
-            JUJU_DIST, prefix=prefix, include='metadata'):
+            JUJU_DIST, prefix=prefix, include=Include.METADATA):
         sync_file = SyncFile(
-            path=blob.name, md5content=blob.properties.content_md5,
+            path=blob.name,
+            md5content=blob.properties.content_settings.content_md5,
             size=blob.properties.content_length,
-            mimetype=blob.properties.content_type, local_path='')
+            mimetype=blob.properties.content_settings.content_type,
+            local_path='')
         files.append(sync_file)
     return sorted(files, key=attrgetter('path'))
 
@@ -123,7 +130,6 @@
     the azure restrictions. The blocks are then assembled into a blob
     with the md5 content (base64 encoded digest).
     """
-    blob_service.put_blob(JUJU_DIST, sync_file.path, '', 'BlockBlob')
     block_ids = []
     index = 0
     with open(sync_file.local_path, 'rb') as local_file:
@@ -135,7 +141,7 @@
                     try:
                         blob_service.put_block(
                             JUJU_DIST, sync_file.path, data, block_id)
-                        block_ids.append(block_id)
+                        block_ids.append(BlobBlock(id=block_id))
                         index += 1
                         break
                     except socket.error as e:
@@ -147,10 +153,12 @@
                 break
     for i in range(0, 3):
         try:
+            content_settings = ContentSettings(
+                content_type=sync_file.mimetype,
+                content_md5=sync_file.md5content)
             blob_service.put_block_list(
                 JUJU_DIST, sync_file.path, block_ids,
-                x_ms_blob_content_type=sync_file.mimetype,
-                x_ms_blob_content_md5=sync_file.md5content)
+                content_settings=content_settings)
             break
         except socket.error as e:
             if e.errno not in (socket.errno.ECONNREFUSED,
@@ -242,7 +250,8 @@
     """Execute the commands from the command line."""
     parser = get_option_parser()
     args = parser.parse_args()
-    blob_service = BlobService()
+    blob_service = BlockBlobService(
+        account_name=args.account_name, account_key=args.account_key)
     if args.command == SYNC:
         return sync_files(blob_service, args.prefix, args.local_dir, args)
     if args.purpose not in PURPOSES:
@@ -264,12 +273,20 @@
     parser.add_argument("-d", "--dry-run", action="store_true",
                         default=False, help=dr_help)
     parser.add_argument('-v', '--verbose', action="store_true",
-                        default=False, help='Increse verbosity.')
+                        default=False, help='Increase verbosity.')
 
 
 def get_option_parser():
     """Return the option parser for this program."""
-    parser = ArgumentParser("Manage objects in Azure blob storage")
+    parser = ArgumentParser(description="Manage objects in Azure blob storage")
+    parser.add_argument(
+        '--account-name',
+        default=os.environ.get('AZURE_STORAGE_ACCOUNT', None),
+        help="The azure storage account, or env AZURE_STORAGE_ACCOUNT.")
+    parser.add_argument(
+        '--account-key',
+        default=os.environ.get('AZURE_STORAGE_ACCESS_KEY', None),
+        help="The azure storage account, or env AZURE_STORAGE_ACCESS_KEY.")
     subparsers = parser.add_subparsers(help='sub-command help', dest='command')
     for command in (LIST, PUBLISH, DELETE):
         subparser = subparsers.add_parser(command, help='Command to run')

=== modified file 'tests/test_azure_publish_tools.py'
--- tests/test_azure_publish_tools.py	2015-10-19 15:10:46 +0000
+++ tests/test_azure_publish_tools.py	2016-06-17 21:21:24 +0000
@@ -4,6 +4,7 @@
 
 from azure_publish_tools import (
     DELETE,
+    delete_files,
     get_option_parser,
     get_local_files,
     get_local_sync_files,
@@ -25,6 +26,12 @@
     write_file,
     )
 
+from azure.storage.blob import (
+    BlobBlock,
+    ContentSettings,
+    Include,
+    )
+
 
 md5sum = {
     'qux': '2FsSE0c8L9fCBFAgprnGKw==',
@@ -41,13 +48,14 @@
     def test_list(self):
         args = self.parse_args(['list', 'mypurpose'])
         self.assertEqual(Namespace(
-            command=LIST, purpose='mypurpose'), args)
+            command=LIST, purpose='mypurpose',
+            account_key=None, account_name=None), args)
 
     def test_publish(self):
         args = self.parse_args(['publish', 'mypurpose', 'mypath'])
         self.assertEqual(Namespace(
             command=PUBLISH, purpose='mypurpose', dry_run=False, verbose=False,
-            path='mypath'), args)
+            path='mypath', account_key=None, account_name=None), args)
 
     def test_publish_dry_run(self):
         args = self.parse_args(['publish', 'mypurpose', 'mypath', '--dry-run'])
@@ -65,7 +73,7 @@
         args = self.parse_args(['delete', 'mypurpose', 'mypath'])
         self.assertEqual(Namespace(
             command=DELETE, purpose='mypurpose', dry_run=False, verbose=False,
-            path=['mypath']), args)
+            path=['mypath'], account_key=None, account_name=None), args)
 
     def test_delete_dry_run(self):
         args = self.parse_args(['delete', 'mypurpose', 'mypath', '--dry-run'])
@@ -83,7 +91,7 @@
         args = self.parse_args(['sync', 'mypath', 'myprefix'])
         self.assertEqual(Namespace(
             command=SYNC, prefix='myprefix', dry_run=False, verbose=False,
-            local_dir='mypath'), args)
+            local_dir='mypath', account_key=None, account_name=None), args)
 
     def test_sync_dry_run(self):
         args = self.parse_args(['sync', 'mypath', 'myprefix', '--dry-run'])
@@ -101,9 +109,9 @@
 class FakeBlobProperties:
 
     def __init__(self, md5, length, content_type):
-        self.content_md5 = md5
         self.content_length = length
-        self.content_type = content_type
+        self.content_settings = ContentSettings(
+            content_type=content_type, content_md5=md5)
 
 
 class FakeBlob:
@@ -130,35 +138,31 @@
         self.containers = {JUJU_DIST: blobs}
 
     def list_blobs(self, container_name, prefix=None, marker=None,
-                   maxresults=None, include=None, delimiter=None):
+                   timeout=None, include=None, delimiter=None):
         if marker is not None:
             raise NotImplementedError('marker not implemented.')
-        if maxresults is not None:
-            raise NotImplementedError('maxresults not implemented.')
-        if include != 'metadata':
-            raise NotImplementedError('include must be "metadata".')
+        if timeout is not None:
+            raise NotImplementedError('timeout not implemented.')
+        if include != Include.METADATA:
+            raise NotImplementedError('include must be "Include.METADATA".')
         if delimiter is not None:
             raise NotImplementedError('delimiter not implemented.')
         return [b for p, b in self.containers[container_name].items()
                 if p.startswith(prefix)]
 
-    def put_blob(self, container_name, blob_name, blob, x_ms_blob_type):
-        if x_ms_blob_type != 'BlockBlob':
-            raise NotImplementedError('x_ms_blob_type not implemented.')
-        if blob != '':
-            raise NotImplementedError('blob not implemented.')
-        self.containers[container_name][blob_name] = FakeBlob(blob_name)
-
     def put_block(self, container_name, blob_name, block, block_id):
-        self.containers[container_name][blob_name]._blocks[block_id] = block
+        if blob_name not in self.containers[container_name]:
+            self.containers[container_name][blob_name] = FakeBlob(blob_name)
+        self.containers[container_name][blob_name]._blocks[
+            BlobBlock(block_id).id] = block
 
     def put_block_list(self, container_name, blob_name, block_list,
-                       content_md5=None, x_ms_blob_content_type=None,
-                       x_ms_blob_content_encoding=None,
-                       x_ms_blob_content_language=None,
-                       x_ms_blob_content_md5=None):
+                       content_settings=None):
         pass
 
+    def delete_blob(self, container_name, blob_name):
+        del self.containers[container_name][blob_name]
+
 
 class TestGetPublishedFiles(QuietTestCase):
 
@@ -238,7 +242,7 @@
         self.assertEqual(['tools/index.json', 'tools/index2.json'],
                          service.containers[JUJU_DIST].keys())
         blob = service.containers[JUJU_DIST]['tools/index2.json']
-        self.assertEqual({'MA==': '{}\n'}, blob._blocks)
+        self.assertEqual({BlobBlock('MA==').id: '{}\n'}, blob._blocks)
 
     def test_same_local_remote(self):
         args = Namespace(verbose=False, dry_run=False)
@@ -270,7 +274,7 @@
         self.assertEqual(['tools/index2.json'],
                          service.containers[JUJU_DIST].keys())
         blob = service.containers[JUJU_DIST]['tools/index2.json']
-        self.assertEqual({'MA==': '{}\n'}, blob._blocks)
+        self.assertEqual({BlobBlock('MA==').id: '{}\n'}, blob._blocks)
 
     def test_different_local_remote_dry_run(self):
         args = Namespace(verbose=False, dry_run=True)
@@ -322,7 +326,7 @@
         self.assertEqual(['tools/index.json', 'tools/index2.json'],
                          service.containers[JUJU_DIST].keys())
         blob = service.containers[JUJU_DIST]['tools/index2.json']
-        self.assertEqual({'MA==': '{}\n'}, blob._blocks)
+        self.assertEqual({BlobBlock('MA==').id: '{}\n'}, blob._blocks)
 
     def test_different_local_remote(self):
         args = Namespace(verbose=False, dry_run=False)
@@ -337,7 +341,7 @@
         self.assertEqual(['tools/index2.json'],
                          service.containers[JUJU_DIST].keys())
         blob = service.containers[JUJU_DIST]['tools/index2.json']
-        self.assertEqual({'MA==': '{}\n'}, blob._blocks)
+        self.assertEqual({BlobBlock('MA==').id: '{}\n'}, blob._blocks)
 
     def test_different_local_remote_dry_run(self):
         args = Namespace(verbose=False, dry_run=True)
@@ -449,3 +453,35 @@
             os.symlink('foo', foo_path)
             result = get_local_sync_files('bools', local_dir)
             self.assertEqual([], result)
+
+
+class TestDeleteFiles(TestCase):
+
+    def test_delete_files(self):
+        args = Namespace(verbose=False, dry_run=False)
+        file1 = SyncFile(
+            'index.json', 33, 'md5-asdf', 'application/json', '')
+        file2 = SyncFile(
+            'other.json', 33, 'md5-asdf', 'application/json', '')
+        blob_service = FakeBlobService({
+            'tools/index.json': FakeBlob.from_sync_file(file1),
+            'tools/other.json': FakeBlob.from_sync_file(file2)
+            })
+        delete_files(blob_service, 'released', ['index.json'], args)
+        self.assertIsNone(
+            blob_service.containers[JUJU_DIST].get('tools/index.json'))
+        self.assertEqual(
+            file2.path,
+            blob_service.containers[JUJU_DIST]['tools/other.json'].name)
+
+    def test_delete_files_dry_run(self):
+        args = Namespace(verbose=False, dry_run=True)
+        file1 = SyncFile(
+            'index.json', 33, 'md5-asdf', 'application/json', '')
+        blob_service = FakeBlobService({
+            'tools/index.json': FakeBlob.from_sync_file(file1),
+            })
+        delete_files(blob_service, 'released', ['index.json'], args)
+        self.assertEqual(
+            file1.path,
+            blob_service.containers[JUJU_DIST]['tools/index.json'].name)