Merge ~barryprice/spade/+git/spade:black into spade:master

[Barry Price]

Reformatted with black

[Haw Loeung (hloeung)]

LGTM

review: Approve (+1)

Unmerged commits

1734965... by Barry Price on 2020-11-27

Merge branch 'flake8' into black

6817bc5... by Barry Price on 2020-11-27

Merge branch 'master' into flake8

c85283a... by Barry Price on 2020-11-27

Typo

a86afa5... by Barry Price on 2020-11-27

Reformatted with black

aa2d273... by Barry Price on 2020-11-27

Address all Flake8 errors bar complexity (noqa: C901)

7e34f18... by Barry Price on 2020-11-27

Modernise, switch to python3 exclusively

diff --git a/Makefile b/Makefile
index 03544a4..ceb744e 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,8 @@ version := $(shell python3 -c "from spade import __version__; print(__version__)
 tempfiles_snap := snap/snapcraft.yaml spade*.snap
 
 blacken:
-	@echo "Normalising python layout with black (but not really)."
+	@echo "Normalising python layout with black"
+	@tox -e black
 
 lint: blacken
 	@echo "Running flake8"
diff --git a/spade/AzureBucket.py b/spade/AzureBucket.py
index 0f7a62d..e04830a 100644
--- a/spade/AzureBucket.py
+++ b/spade/AzureBucket.py
@@ -26,15 +26,14 @@ from spade.ObjectIO import ObjectIO
 
 DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
 DEFAULT_DELIMITER = "/"
-AZURE_SEGMENTSIZE = 1024 * 1024 * 4   # Azure block (segment) limit = 4MB
+AZURE_SEGMENTSIZE = 1024 * 1024 * 4  # Azure block (segment) limit = 4MB
 MAX_AZURE_OBJECTSIZE = 64 * 1024 * 1024  # Azure block blob limit = 64MB
 USER_META_PREFIX = "x-ms-meta-"
 
 
-class AzureBucket():
+class AzureBucket:
     def __init__(self, creds):
-        wanted = ["private_key",
-                  "storage_account"]
+        wanted = ["private_key", "storage_account"]
         for k in wanted:
             if k not in creds:
                 # Missing creds
@@ -67,9 +66,19 @@ class AzureBucket():
     def _create_azure_auth_signature(self, bucket, headers, query, method):
         # Note: auth_headers are in a specific order
         # see https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
-        auth_headers = ("Content-Encoding", "Content-Language", "Content-Length", "Content-MD5",
-                        "Content-Type", "Date", "If-Modified-Since", "If-Match", "If-None-Match",
-                        "If-Unmodified-Since", "Range")
+        auth_headers = (
+            "Content-Encoding",
+            "Content-Language",
+            "Content-Length",
+            "Content-MD5",
+            "Content-Type",
+            "Date",
+            "If-Modified-Since",
+            "If-Match",
+            "If-None-Match",
+            "If-Unmodified-Since",
+            "Range",
+        )
         canonicalized_headers = self._dict_to_str(headers, strip_final=False, startswith="x-ms-")
         query_string = self._dict_to_str(query)
 
@@ -84,9 +93,11 @@ class AzureBucket():
         signature += canonicalized_headers
         signature += canonicalized_resource
 
-        encoded_sig = base64.b64encode(hmac.new(base64.b64decode(self.primaryKey),
-                                                msg=signature.encode("utf-8"),
-                                                digestmod=hashlib.sha256).digest())
+        encoded_sig = base64.b64encode(
+            hmac.new(
+                base64.b64decode(self.primaryKey), msg=signature.encode("utf-8"), digestmod=hashlib.sha256
+            ).digest()
+        )
         return encoded_sig.decode('utf-8')
 
     # Requests documented here: https://msdn.microsoft.com/en-GB/library/azure/dd135733.aspx
@@ -104,10 +115,9 @@ class AzureBucket():
         headers["Authorization"] = "SharedKey {}:{}".format(self.storageAccount, auth)
 
         query_string = self._dict_to_str(query, kv_sep="=", entry_sep="&")
-        response, content = h.request(uri=self.blobUri + bucket + "?" + query_string,
-                                      method=method,
-                                      body=body,
-                                      headers=headers)
+        response, content = h.request(
+            uri=self.blobUri + bucket + "?" + query_string, method=method, body=body, headers=headers
+        )
         return response, content
 
     def listBuckets(self):
@@ -153,9 +163,7 @@ class AzureBucket():
 
     def getBucketMetadata(self, bucketName):
         # Standard metadata
-        response, content = self._send_azure_request(bucketName,
-                                                     query={"restype": "container"},
-                                                     method="HEAD")
+        response, content = self._send_azure_request(bucketName, query={"restype": "container"}, method="HEAD")
         if response.get("status") == "200":
             co = BucketObject(bucketName)
             co.size = int(response.get("content-length", 0))
@@ -169,9 +177,7 @@ class AzureBucket():
 
         # User-defined metadata
         query = {"restype": "container", "comp": "metadata"}
-        response, content = self._send_azure_request(bucketName,
-                                                     query=query,
-                                                     method="HEAD")
+        response, content = self._send_azure_request(bucketName, query=query, method="HEAD")
         if response.get("status") == "200":
             for k, v in response.items():
                 if k.lower().startswith(USER_META_PREFIX):
@@ -187,8 +193,7 @@ class AzureBucket():
 
     def getObjectMetadata(self, bucketName, objectName):
         # Standard metadata
-        response, content = self._send_azure_request(bucketName + "/" + objectName,
-                                                     method="HEAD")
+        response, content = self._send_azure_request(bucketName + "/" + objectName, method="HEAD")
         if response.get("status") == "200":
             co = BucketObject(objectName)
             co.size = int(response.get("content-length", 0))
@@ -201,9 +206,9 @@ class AzureBucket():
             raise BucketError("Azure Failed to find object metadata")
 
         # User-defined metadata
-        response, content = self._send_azure_request(bucketName + "/" + objectName,
-                                                     query={"comp": "metadata"},
-                                                     method="HEAD")
+        response, content = self._send_azure_request(
+            bucketName + "/" + objectName, query={"comp": "metadata"}, method="HEAD"
+        )
         if response.get("status") == "200":
             for k, v in response.items():
                 if k.lower().startswith(USER_META_PREFIX):
@@ -225,9 +230,13 @@ class AzureBucket():
         return metadata.rawMetadata.get("etag", "")
 
     def setObjectMetadata(self, bucketName, objectName, metadata, overwrite=False):
-        systemMetadata = ["x-ms-blob-cache-control", "x-ms-blob-content-type",
-                          "x-ms-blob-content-md5", "x-ms-blob-content-encoding",
-                          "x-ms-blob-content-language"]
+        systemMetadata = [
+            "x-ms-blob-cache-control",
+            "x-ms-blob-content-type",
+            "x-ms-blob-content-md5",
+            "x-ms-blob-content-encoding",
+            "x-ms-blob-content-language",
+        ]
 
         newStandardMetadata = {}
         newUserMetadata = {}
@@ -252,18 +261,16 @@ class AzureBucket():
                 newUserMetadata[USER_META_PREFIX + k] = v
 
         # Standard metadata
-        response, content = self._send_azure_request(bucketName + "/" + objectName,
-                                                     query={"comp": "properties"},
-                                                     headers=newStandardMetadata,
-                                                     method="PUT")
+        response, content = self._send_azure_request(
+            bucketName + "/" + objectName, query={"comp": "properties"}, headers=newStandardMetadata, method="PUT"
+        )
         if response.get("status") != "200":
             raise BucketError("Azure Failed to set standard object metadata")
 
         # User metadata
-        response, content = self._send_azure_request(bucketName + "/" + objectName,
-                                                     query={"comp": "metadata"},
-                                                     headers=newUserMetadata,
-                                                     method="PUT")
+        response, content = self._send_azure_request(
+            bucketName + "/" + objectName, query={"comp": "metadata"}, headers=newUserMetadata, method="PUT"
+        )
         if response.get("status") != "200":
             raise BucketError("Azure Failed to set user object metadata")
         return
@@ -272,8 +279,7 @@ class AzureBucket():
         headers = {}
         if rangeStart is not None and rangeEnd is not None:
             headers["Range"] = "bytes={}-{}".format(rangeStart, rangeEnd)
-        response, content = self._send_azure_request(bucketName + "/" + objectName,
-                                                     headers=headers)
+        response, content = self._send_azure_request(bucketName + "/" + objectName, headers=headers)
         if response.get("status") == "200" or response.get("status") == "206":
             return content
         elif response.get("status") == "416":
@@ -293,11 +299,13 @@ class AzureBucket():
             blockId = base64.b64encode("{:06}".format(blockNumber).encode('utf-8')).decode('utf-8')
             headers = {}
             headers["Content-Length"] = str(len(blockContent))
-            response, content = self._send_azure_request(bucketName + "/" + objectName,
-                                                         query={"comp": "block", "blockid": blockId},
-                                                         headers=headers,
-                                                         body=blockContent,
-                                                         method="PUT")
+            response, content = self._send_azure_request(
+                bucketName + "/" + objectName,
+                query={"comp": "block", "blockid": blockId},
+                headers=headers,
+                body=blockContent,
+                method="PUT",
+            )
             if not response.get("status") == "201":
                 raise BucketError("Azure Failed to upload multi-part block")
             blockList.append(blockId)
@@ -312,11 +320,9 @@ class AzureBucket():
         xml += "</BlockList>\n"
         headers = {}
         headers["Content-Length"] = str(len(xml))
-        response, content = self._send_azure_request(bucketName + "/" + objectName,
-                                                     query={"comp": "blocklist"},
-                                                     headers=headers,
-                                                     body=xml,
-                                                     method="PUT")
+        response, content = self._send_azure_request(
+            bucketName + "/" + objectName, query={"comp": "blocklist"}, headers=headers, body=xml, method="PUT"
+        )
         if not response.get("status") == "201":
             raise BucketError("Azure Failed to upload blocklist")
         return
@@ -329,10 +335,9 @@ class AzureBucket():
         headers["Content-Length"] = str(size)
         headers["Content-Type"] = "application/octet-stream"
         headers["x-ms-blob-type"] = "BlockBlob"
-        response, content = self._send_azure_request(bucketName + "/" + objectName,
-                                                     headers=headers,
-                                                     body=src,
-                                                     method="PUT")
+        response, content = self._send_azure_request(
+            bucketName + "/" + objectName, headers=headers, body=src, method="PUT"
+        )
         if response.get("status") == "201":
             return
         else:
@@ -342,8 +347,7 @@ class AzureBucket():
         return ObjectIO(self, bucketName, objectName, blockSize)
 
     def deleteObject(self, bucketName, objectName):
-        response, content = self._send_azure_request(bucketName + "/" + objectName,
-                                                     method="DELETE")
+        response, content = self._send_azure_request(bucketName + "/" + objectName, method="DELETE")
         if response.get("status") == "202":
             return
         else:
@@ -351,9 +355,7 @@ class AzureBucket():
 
     def createBucket(self, bucketName):
         query = {"restype": "container"}
-        response, content = self._send_azure_request(bucketName,
-                                                     query=query,
-                                                     method="PUT")
+        response, content = self._send_azure_request(bucketName, query=query, method="PUT")
         if response.get("status") == "201":
             return
         else:
@@ -362,9 +364,7 @@ class AzureBucket():
     # Note: Azure doesn't care if the container is empty or not
     def deleteBucket(self, bucketName):
         query = {"restype": "container"}
-        response, content = self._send_azure_request(bucketName,
-                                                     query=query,
-                                                     method="DELETE")
+        response, content = self._send_azure_request(bucketName, query=query, method="DELETE")
         if response.get("status") == "202":
             return
         else:
@@ -378,9 +378,7 @@ class AzureBucket():
             return []
 
         query = {"restype": "container", "comp": "acl"}
-        response, content = self._send_azure_request(bucketName,
-                                                     query=query,
-                                                     method="HEAD")
+        response, content = self._send_azure_request(bucketName, query=query, method="HEAD")
         if response.get("status") != "200":
             raise BucketError("Azure Failed to read ACL data")
 
diff --git a/spade/BucketObject.py b/spade/BucketObject.py
index 9f8a258..7d35d24 100644
--- a/spade/BucketObject.py
+++ b/spade/BucketObject.py
@@ -1,5 +1,4 @@
-class BucketObject():
-
+class BucketObject:
     class ObjectType:
         File = 0
         Directory = 1
@@ -18,7 +17,7 @@ class BucketError(Exception):
     pass
 
 
-class BucketAcl():
+class BucketAcl:
     def __init__(self):
         self.acls = {}
 
diff --git a/spade/GCEBucket.py b/spade/GCEBucket.py
index 2029886..56761f1 100644
--- a/spade/GCEBucket.py
+++ b/spade/GCEBucket.py
@@ -23,6 +23,7 @@ import datetime
 import base64
 from apiclient import discovery, errors
 from apiclient.http import MediaIoBaseUpload
+
 try:
     from oauth2client.service_account import ServiceAccountCredentials
 except ImportError:
@@ -34,11 +35,9 @@ DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
 DEFAULT_DELIMITER = "/"
 
 
-class GCEBucket():
+class GCEBucket:
     def __init__(self, creds):
-        wanted = ["client_email",
-                  "private_key",
-                  "project"]
+        wanted = ["client_email", "private_key", "project"]
         for k in wanted:
             if k not in creds:
                 # Missing creds
@@ -47,12 +46,9 @@ class GCEBucket():
         # httplib2.debuglevel = 4
         cloud_storage_auth_scope = 'https://www.googleapis.com/auth/devstorage.full_control'
         try:
-            c = ServiceAccountCredentials.from_json_keyfile_dict(creds,
-                                                                 scopes=cloud_storage_auth_scope)
+            c = ServiceAccountCredentials.from_json_keyfile_dict(creds, scopes=cloud_storage_auth_scope)
         except NameError:
-            c = SignedJwtAssertionCredentials(creds['client_email'],
-                                              creds['private_key'],
-                                              cloud_storage_auth_scope)
+            c = SignedJwtAssertionCredentials(creds['client_email'], creds['private_key'], cloud_storage_auth_scope)
         http = c.authorize(httplib2.Http())
         self.gce = discovery.build('storage', 'v1', http=http)
         self.project = creds["project"]
@@ -79,10 +75,7 @@ class GCEBucket():
     def listBucketContents(self, bucketName, path=None, delimiter=DEFAULT_DELIMITER):
         contents = []
         fields_to_return = 'nextPageToken,prefixes,items'
-        req = self.gce.objects().list(bucket=bucketName,
-                                      prefix=path,
-                                      delimiter=delimiter,
-                                      fields=fields_to_return)
+        req = self.gce.objects().list(bucket=bucketName, prefix=path, delimiter=delimiter, fields=fields_to_return)
         while req:
             try:
                 resp = req.execute()
@@ -106,8 +99,7 @@ class GCEBucket():
         return contents
 
     def getBucketMetadata(self, bucketName):
-        req = self.gce.buckets().get(bucket=bucketName,
-                                     projection="full")
+        req = self.gce.buckets().get(bucket=bucketName, projection="full")
         try:
             resp = req.execute()
         except errors.HttpError:
@@ -126,9 +118,7 @@ class GCEBucket():
         raise BucketError("GCE Setting bucket metadata is not supported in GCE")
 
     def getObjectMetadata(self, bucketName, objectName):
-        req = self.gce.objects().get(bucket=bucketName,
-                                     object=objectName,
-                                     projection="full")
+        req = self.gce.objects().get(bucket=bucketName, object=objectName, projection="full")
         try:
             resp = req.execute()
         except errors.HttpError:
@@ -157,8 +147,7 @@ class GCEBucket():
         return hexChecksum
 
     def setObjectMetadata(self, bucketName, objectName, metadata, overwrite=False):
-        systemMetadata = ["cacheControl", "contentDisposition", "contentEncoding",
-                          "contentLanguage", "contentType"]
+        systemMetadata = ["cacheControl", "contentDisposition", "contentEncoding", "contentLanguage", "contentType"]
         newMetadata = {}
         m = self.getObjectMetadata(bucketName, objectName)
         # Keep existing acls, system metadata and (optionally) user metadata
@@ -177,9 +166,7 @@ class GCEBucket():
                 newMetadata[k] = v
             else:
                 newMetadata["metadata"][k] = v
-        req = self.gce.objects().update(bucket=bucketName,
-                                        object=objectName,
-                                        body=newMetadata)
+        req = self.gce.objects().update(bucket=bucketName, object=objectName, body=newMetadata)
         try:
             req.execute()
         except errors.HttpError:
@@ -190,22 +177,18 @@ class GCEBucket():
         headers = {}
         if rangeStart is not None and rangeEnd is not None:
             headers['range'] = "bytes={}-{}".format(rangeStart, rangeEnd)
-        req = self.gce.objects().get_media(bucket=bucketName,
-                                           object=objectName)
+        req = self.gce.objects().get_media(bucket=bucketName, object=objectName)
         http = req.http
         resp, data = http.request(req.uri, headers=headers)
         if resp.get("status") == "416":
             # Requested range not satisfiable
             data = ""
         elif resp.get("status")[0] != "2":
-            raise(BucketError("GCE Failed to find object"))
+            raise (BucketError("GCE Failed to find object"))
         return data
 
     def putObject(self, bucketName, objectName, src, size, blockSize=DEFAULT_BLOCKSIZE):
-        media = MediaIoBaseUpload(src,
-                                  mimetype="application/octet-stream",
-                                  chunksize=blockSize,
-                                  resumable=True)
+        media = MediaIoBaseUpload(src, mimetype="application/octet-stream", chunksize=blockSize, resumable=True)
         req = self.gce.objects().insert(bucket=bucketName, name=objectName, media_body=media)
         resp = None
         while resp is None:
@@ -243,9 +226,7 @@ class GCEBucket():
         return
 
     aclTypes = ["list", "read", "write"]
-    aclTypeMap = {"read": "READER",
-                  "write": "WRITER",
-                  "list": "READER"}
+    aclTypeMap = {"read": "READER", "write": "WRITER", "list": "READER"}
 
     def translateAcl(self, aclData, aliases):
         """
diff --git a/spade/LocalBucket.py b/spade/LocalBucket.py
index f2293ec..9f3f974 100644
--- a/spade/LocalBucket.py
+++ b/spade/LocalBucket.py
@@ -10,7 +10,7 @@ DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
 DEFAULT_DELIMITER = "/"
 
 
-class LocalBucket():
+class LocalBucket:
     def __init__(self, creds):
         wanted = ["bucket_root"]
         for k in wanted:
@@ -79,8 +79,20 @@ class LocalBucket():
         co.date = self.standardTimestamp(stat.st_mtime)
         co.contentType = "text/plain"
         raw = {}
-        for num, name in enumerate(["st_mode", "st_ino", "st_dev", "st_nlink", "st_uid",
-                                    "st_gid", "st_size", "st_atime", "st_mtime", "st_ctime"]):
+        for num, name in enumerate(
+            [
+                "st_mode",
+                "st_ino",
+                "st_dev",
+                "st_nlink",
+                "st_uid",
+                "st_gid",
+                "st_size",
+                "st_atime",
+                "st_mtime",
+                "st_ctime",
+            ]
+        ):
             raw[name] = stat[num]
         co.rawMetadata = raw
         return co
diff --git a/spade/ObjectIO.py b/spade/ObjectIO.py
index fc2b38f..dfdb804 100644
--- a/spade/ObjectIO.py
+++ b/spade/ObjectIO.py
@@ -2,7 +2,7 @@ import os
 from spade.BucketObject import BucketError
 
 
-class ObjectIO():
+class ObjectIO:
     def __init__(self, parent, bucketName, objectName, blockSize=-1):
         self.parent = parent
         self.bucketName = bucketName
@@ -36,10 +36,7 @@ class ObjectIO():
         if self.closed:
             raise ValueError
 
-        data = self.parent.getObject(self.bucketName,
-                                     self.objectName,
-                                     self.pos,
-                                     min(self.pos + n - 1, self.len))
+        data = self.parent.getObject(self.bucketName, self.objectName, self.pos, min(self.pos + n - 1, self.len))
         self.pos = self.pos + len(data)
         return data
 
@@ -56,19 +53,19 @@ class ObjectIO():
         while self.readlinePos < len(self.readlineBuffer):
             lineEndPos = self.readlineBuffer.find(b"\n", self.readlinePos)
             if lineEndPos >= 0:
-                line = self.readlineBuffer[self.readlinePos:lineEndPos + 1].decode('utf-8')
+                line = self.readlineBuffer[self.readlinePos : lineEndPos + 1].decode('utf-8')
                 self.readlinePos = lineEndPos + 1
                 return line
             else:
                 newData = self.read(self.blockSize)
                 if not newData:
                     return
-                self.readlineBuffer = self.readlineBuffer[self.readlinePos:] + newData
+                self.readlineBuffer = self.readlineBuffer[self.readlinePos :] + newData
                 self.readlinePos = 0
 
         # In case the last line has no newline
         if self.readlinePos < len(self.readlineBuffer):
-            return self.readlineBuffer[self.readlinePos:]
+            return self.readlineBuffer[self.readlinePos :]
 
     def readlines(self):
         while True:
diff --git a/spade/S3Bucket.py b/spade/S3Bucket.py
index 6d3036e..67f38fb 100644
--- a/spade/S3Bucket.py
+++ b/spade/S3Bucket.py
@@ -25,15 +25,13 @@ from spade.ObjectIO import ObjectIO
 DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
 DEFAULT_DELIMITER = "/"
 MAX_S3_OBJECTSIZE = 5 * 1024 * 1024 * 1024  # S3 object PUT limit = 5GB
-S3_SEGMENTSIZE = 1024 * 1024 * 1024         # Use 1GB segments for large object uploads
+S3_SEGMENTSIZE = 1024 * 1024 * 1024  # Use 1GB segments for large object uploads
 USER_META_PREFIX = "x-amz-meta-"
 
 
-class S3Bucket():
+class S3Bucket:
     def __init__(self, creds):
-        wanted = ["aws_access_key_id",
-                  "aws_secret_access_key",
-                  "aws_region"]
+        wanted = ["aws_access_key_id", "aws_secret_access_key", "aws_region"]
         for k in wanted:
             if k not in creds:
                 # Missing creds
@@ -44,8 +42,9 @@ class S3Bucket():
 
     def oldLibraryFixes(self):
         try:
-            self.regions = [{"name": region.name, "endpoint": region.endpoint}
-                            for region in boto.regioninfo.get_regions("s3")]
+            self.regions = [
+                {"name": region.name, "endpoint": region.endpoint} for region in boto.regioninfo.get_regions("s3")
+            ]
         except AttributeError:
             # Old boto library doesn't have this, so we'll use the ones we know about
             self.regions = [
@@ -63,9 +62,22 @@ class S3Bucket():
                 {'name': 'us-east-1', 'endpoint': 's3.amazonaws.com'},
             ]
 
-            self.base_fields = set(['content-length', 'content-language', 'content-disposition',
-                                    'content-encoding', 'expires', 'content-md5', 'last-modified',
-                                    'etag', 'cache-control', 'date', 'content-type', 'x-robots-tag'])
+            self.base_fields = set(
+                [
+                    'content-length',
+                    'content-language',
+                    'content-disposition',
+                    'content-encoding',
+                    'expires',
+                    'content-md5',
+                    'last-modified',
+                    'etag',
+                    'cache-control',
+                    'date',
+                    'content-type',
+                    'x-robots-tag',
+                ]
+            )
 
     def _connect(self):
         self.endpoint = None
@@ -78,11 +90,13 @@ class S3Bucket():
         if not self.endpoint:
             raise NotImplementedError
 
-        self.s3 = boto.s3.connection.S3Connection(self.creds['aws_access_key_id'],
-                                                  self.creds['aws_secret_access_key'],
-                                                  is_secure=True,
-                                                  host=self.endpoint,
-                                                  calling_format=OrdinaryCallingFormat())
+        self.s3 = boto.s3.connection.S3Connection(
+            self.creds['aws_access_key_id'],
+            self.creds['aws_secret_access_key'],
+            is_secure=True,
+            host=self.endpoint,
+            calling_format=OrdinaryCallingFormat(),
+        )
 
     # Boto/S3 gives a 301 redirect response if we try to access a bucket
     # in another region, so make sure we're in the right one
@@ -94,7 +108,7 @@ class S3Bucket():
 
         # Strip off the first section (the current region)
         try:
-            websiteEndpoint = websiteEndpoint[websiteEndpoint.find(".") + 1:]
+            websiteEndpoint = websiteEndpoint[websiteEndpoint.find(".") + 1 :]
         except UnboundLocalError:
             raise BucketError("S3 Failed to find bucket")
 
@@ -176,10 +190,12 @@ class S3Bucket():
         return metadata.rawMetadata.get("etag", "").strip('"')
 
     def setObjectMetadata(self, bucketName, objectName, metadata, overwrite=False):
-        systemMetadata = ["x-amz-server-side-encryption",
-                          "x-amz-storage-class",
-                          "x-amz-website-redirect-location",
-                          "x-amz-server-side-encryption-aws-kms-key-id"]
+        systemMetadata = [
+            "x-amz-server-side-encryption",
+            "x-amz-storage-class",
+            "x-amz-website-redirect-location",
+            "x-amz-server-side-encryption-aws-kms-key-id",
+        ]
         self.confirmRegion(bucketName)
 
         # Metadata to remove (if any)
@@ -300,19 +316,17 @@ class S3Bucket():
         return
 
     aclTypes = ["list", "read", "write", "readacl", "writeacl"]
-    aclTypeMap = {"read": "READ",
-                  "write": "WRITE",
-                  "list": "READ",
-                  "readacl": "READ_ACP",
-                  "writeacl": "WRITE_ACP"}
+    aclTypeMap = {"read": "READ", "write": "WRITE", "list": "READ", "readacl": "READ_ACP", "writeacl": "WRITE_ACP"}
 
     def createAclData(self, user, type):
-        aclData = {"s3_display_name": None,
-                   "s3_id": user,
-                   "s3_uri": None,
-                   "s3_email_address": None,
-                   "s3_type": "CanonicalUser",
-                   "s3_permission": self.aclTypeMap[type]}
+        aclData = {
+            "s3_display_name": None,
+            "s3_id": user,
+            "s3_uri": None,
+            "s3_email_address": None,
+            "s3_type": "CanonicalUser",
+            "s3_permission": self.aclTypeMap[type],
+        }
         return aclData
 
     def translateAcl(self, aclData, aliases):
@@ -343,12 +357,14 @@ class S3Bucket():
 
         acls = BucketAcl()
         for grant in acl.acl.grants:
-            aclData = {"s3_display_name": grant.display_name,
-                       "s3_id": grant.id,
-                       "s3_uri": grant.uri,
-                       "s3_email_address": grant.email_address,
-                       "s3_type": grant.type,
-                       "s3_permission": grant.permission}
+            aclData = {
+                "s3_display_name": grant.display_name,
+                "s3_id": grant.id,
+                "s3_uri": grant.uri,
+                "s3_email_address": grant.email_address,
+                "s3_type": grant.type,
+                "s3_permission": grant.permission,
+            }
             if grant.permission in ["READ", "FULL_CONTROL"]:
                 if objectName:
                     acls.add("read", aclData)
@@ -392,12 +408,14 @@ class S3Bucket():
                     uniqueAcls.append(user)
 
         for user in uniqueAcls:
-            grant = boto.s3.acl.Grant(permission=user["s3_permission"],
-                                      type=user["s3_type"],
-                                      id=user["s3_id"],
-                                      display_name=user["s3_display_name"],
-                                      uri=user["s3_uri"],
-                                      email_address=["s3_email_address"])
+            grant = boto.s3.acl.Grant(
+                permission=user["s3_permission"],
+                type=user["s3_type"],
+                id=user["s3_id"],
+                display_name=user["s3_display_name"],
+                uri=user["s3_uri"],
+                email_address=["s3_email_address"],
+            )
             s3acl.acl.add_grant(grant)
 
         try:
diff --git a/spade/Spade.py b/spade/Spade.py
index b8f4b12..5ebdfc0 100644
--- a/spade/Spade.py
+++ b/spade/Spade.py
@@ -6,21 +6,25 @@ import re
 import json
 import fnmatch
 from spade.BucketObject import BucketObject, BucketError
+
 try:
     from urllib.parse import urlparse
 except ImportError:
     from urlparse import urlparse
 
 import gzip
+
 compressionMethods = ["gzip"]
 if sys.version_info.major >= 3 and sys.version_info.minor >= 3:
     import bz2
+
     compressionMethods.append("bzip2")
     import lzma
+
     compressionMethods.append("xz")
 else:
     # Define a fake lzma.LZMAError
-    class lzma():
+    class lzma:
         class LZMAError(Exception):
             pass
 
@@ -29,7 +33,7 @@ DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
 DEFAULT_DELIMITER = "/"
 
 
-class Result():
+class Result:
     def __init__(self):
         self.output = []
         self.error = None
@@ -38,7 +42,7 @@ class Result():
         return "<Result object.  output: '{}' error: '{}'>".format(self.output, self.error)
 
 
-class Bucket():
+class Bucket:
     def __init__(self, configFile):
         self.configFile = configFile
         self.config = json.loads(open(configFile, 'r').read())
@@ -102,21 +106,27 @@ class Bucket():
 
         if bucketType == "swift":
             from spade.SwiftBucket import SwiftBucket
+
             return SwiftBucket(self.creds[scheme])
         elif bucketType == "s3":
             from spade.S3Bucket import S3Bucket
+
             return S3Bucket(self.creds[scheme])
         elif bucketType == "gce":
             from spade.GCEBucket import GCEBucket
+
             return GCEBucket(self.creds[scheme])
         elif bucketType == "azure":
             from spade.AzureBucket import AzureBucket
+
             return AzureBucket(self.creds[scheme])
         elif bucketType == "local":
             from spade.LocalBucket import LocalBucket
+
             return LocalBucket(self.creds[scheme])
         elif bucketType == "stdio":
             from spade.StdioBucket import StdioBucket
+
             return StdioBucket(self.creds[scheme])
         else:
             raise NotImplementedError
@@ -449,11 +459,7 @@ class Bucket():
                 ret.error = e
                 yield ret
             try:
-                dstBucket.putObject(dstUrl.netloc,
-                                    dstObjName,
-                                    fSrc,
-                                    objMetadata.size,
-                                    blockSize=args.blockSize)
+                dstBucket.putObject(dstUrl.netloc, dstObjName, fSrc, objMetadata.size, blockSize=args.blockSize)
             except BucketError as e:
                 ret.error = e
                 fSrc.close()
diff --git a/spade/StdioBucket.py b/spade/StdioBucket.py
index ef6cb43..31ba1f6 100644
--- a/spade/StdioBucket.py
+++ b/spade/StdioBucket.py
@@ -8,7 +8,7 @@ from spade.ObjectIO import ObjectIO
 DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
 
 
-class StdioBucket():
+class StdioBucket:
     def __init__(self, creds):
         return
 
diff --git a/spade/SwiftBucket.py b/spade/SwiftBucket.py
index 2125b85..0c3adb0 100644
--- a/spade/SwiftBucket.py
+++ b/spade/SwiftBucket.py
@@ -23,19 +23,15 @@ from spade.ObjectIO import ObjectIO
 
 DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
 DEFAULT_DELIMITER = "/"
-MAX_SWIFT_OBJECTSIZE = 5 * 1024 * 1024 * 1024   # Swift object limit = 5GB
-SWIFT_SEGMENTSIZE = 1024 * 1024 * 1024          # Use 1GB segments for large object uploads
+MAX_SWIFT_OBJECTSIZE = 5 * 1024 * 1024 * 1024  # Swift object limit = 5GB
+SWIFT_SEGMENTSIZE = 1024 * 1024 * 1024  # Use 1GB segments for large object uploads
 USER_META_PREFIX = "x-object-meta-"
 ACL_META_PREFIX = "x-container-"
 
 
-class SwiftBucket():
+class SwiftBucket:
     def __init__(self, creds):
-        wanted = ["os_username",
-                  "os_password",
-                  "os_tenant_name",
-                  "os_auth_url",
-                  "os_region_name"]
+        wanted = ["os_username", "os_password", "os_tenant_name", "os_auth_url", "os_region_name"]
         for k in wanted:
             if k not in creds:
                 # Missing creds
@@ -51,7 +47,8 @@ class SwiftBucket():
             key=creds["os_password"],
             retries=5,
             auth_version="2.0",
-            os_options=options)
+            os_options=options,
+        )
 
     def standardTimestamp(self, timestamp):
         try:
@@ -109,8 +106,7 @@ class SwiftBucket():
         return co
 
     def setBucketMetadata(self, bucketName, metadata, overwrite=False):  # noqa: C901
-        systemMetadata = ["content-type", "x-container-bytes-used",
-                          "x-container-object-count:", "x-timestamp"]
+        systemMetadata = ["content-type", "x-container-bytes-used", "x-container-object-count:", "x-timestamp"]
         newMetadata = {}
         # Keep existing system metadata and (optionally) user metadata
         m = self.getBucketMetadata(bucketName)
@@ -163,8 +159,14 @@ class SwiftBucket():
         return metadata.rawMetadata.get("etag", "").strip('"')
 
     def setObjectMetadata(self, bucketName, objectName, metadata, overwrite=False):
-        systemMetadata = ["content-type", "content-encoding", "content-disposition",
-                          "x-delete-at", "x-delete-after", "etag"]
+        systemMetadata = [
+            "content-type",
+            "content-encoding",
+            "content-disposition",
+            "x-delete-at",
+            "x-delete-after",
+            "etag",
+        ]
         newMetadata = {}
         # Keep existing system metadata and (optionally) user metadata
         m = self.getObjectMetadata(bucketName, objectName)
@@ -223,11 +225,13 @@ class SwiftBucket():
             segmentName = '{}/{}/{}/{}/{:08d}'.format(objectName, mtime, size, SWIFT_SEGMENTSIZE, segmentId)
             segmentId += 1
             try:
-                self.swift.put_object(segmentBucketName,
-                                      segmentName,
-                                      src,
-                                      content_length=segmentSize,
-                                      content_type="application/octet-stream")
+                self.swift.put_object(
+                    segmentBucketName,
+                    segmentName,
+                    src,
+                    content_length=segmentSize,
+                    content_type="application/octet-stream",
+                )
             except swiftclient.ClientException:
                 raise BucketError("Swift Failed to put large object segment")
             bytesRead += segmentSize
@@ -239,12 +243,9 @@ class SwiftBucket():
             'x-object-meta-mtime': mtime,
         }
         try:
-            self.swift.put_object(bucketName,
-                                  objectName,
-                                  '',
-                                  content_length=0,
-                                  content_type="application/octet-stream",
-                                  headers=headers)
+            self.swift.put_object(
+                bucketName, objectName, '', content_length=0, content_type="application/octet-stream", headers=headers
+            )
         except swiftclient.ClientException:
             raise BucketError("Swift Failed to put large object manifest")
         return
@@ -254,10 +255,7 @@ class SwiftBucket():
             return self.putLargeObject(bucketName, objectName, src, size, min(blockSize, SWIFT_SEGMENTSIZE))
 
         try:
-            self.swift.put_object(bucketName,
-                                  objectName,
-                                  src,
-                                  content_type="application/octet-stream")
+            self.swift.put_object(bucketName, objectName, src, content_type="application/octet-stream")
         except swiftclient.exceptions.ClientException:
             raise BucketError("Swift Failed to put object")
         return