Merge ~mthaddon/spade/+git/spade:master into spade:master

Proposed by Tom Haddon
Status: Merged
Merged at revision: 81a8c8a513479b566d273d62bef668f89cabe6f8
Proposed branch: ~mthaddon/spade/+git/spade:master
Merge into: spade:master
Diff against target: 348 lines (+37/-25)
11 files modified
Makefile (+3/-0)
spade/AzureBucket.py (+2/-1)
spade/BucketObject.py (+5/-2)
spade/GCEBucket.py (+2/-2)
spade/LocalBucket.py (+3/-3)
spade/ObjectIO.py (+1/-1)
spade/S3Bucket.py (+6/-4)
spade/Spade.py (+7/-6)
spade/StdioBucket.py (+3/-3)
spade/SwiftBucket.py (+4/-3)
spade/spade.py (+1/-0)
Reviewer Review Type Date Requested Status
David Lawson (community) Approve
Stuart Bishop (community) Approve
Review via email: mp+333698@code.launchpad.net

Description of the change

Add a lint target and fix lint warnings

To post a comment you must log in.
Revision history for this message
Stuart Bishop (stub) wrote :

LGTM

review: Approve
Revision history for this message
David Lawson (deej) wrote :

+1

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/Makefile b/Makefile
2index 13d0f12..c3b3ca6 100644
3--- a/Makefile
4+++ b/Makefile
5@@ -18,3 +18,6 @@ snap_clean:
6 rm -Rf $(tempfiles_snap)
7
8 clean: whl_clean snap_clean
9+
10+lint:
11+ @find spade -name \*py | xargs flake8 --config=/dev/null --ignore=C901 --max-line-length=120 --max-complexity=10 --hang-closing
12diff --git a/spade/AzureBucket.py b/spade/AzureBucket.py
13index e600360..92bf430 100644
14--- a/spade/AzureBucket.py
15+++ b/spade/AzureBucket.py
16@@ -30,6 +30,7 @@ AZURE_SEGMENTSIZE = 1024 * 1024 * 4 # Azure block (segment) limit = 4MB
17 MAX_AZURE_OBJECTSIZE = 64 * 1024 * 1024 # Azure block blob limit = 64MB
18 USER_META_PREFIX = "x-ms-meta-"
19
20+
21 class AzureBucket():
22 def __init__(self, creds):
23 wanted = ["private_key",
24@@ -219,7 +220,7 @@ class AzureBucket():
25 def getObjectChecksum(self, bucketName, objectName):
26 try:
27 metadata = self.getObjectMetadata(bucketName, objectName)
28- except BucketError as e:
29+ except BucketError:
30 return ""
31 return metadata.rawMetadata.get("etag", "")
32
33diff --git a/spade/BucketObject.py b/spade/BucketObject.py
34index 5e522b8..9f8a258 100644
35--- a/spade/BucketObject.py
36+++ b/spade/BucketObject.py
37@@ -1,7 +1,8 @@
38 class BucketObject():
39+
40 class ObjectType:
41- File=0
42- Directory=1
43+ File = 0
44+ Directory = 1
45
46 def __init__(self, name):
47 self.name = name
48@@ -12,9 +13,11 @@ class BucketObject():
49 self.userMetadata = {}
50 self.rawMetadata = {}
51
52+
53 class BucketError(Exception):
54 pass
55
56+
57 class BucketAcl():
58 def __init__(self):
59 self.acls = {}
60diff --git a/spade/GCEBucket.py b/spade/GCEBucket.py
61index 23af539..2029886 100644
62--- a/spade/GCEBucket.py
63+++ b/spade/GCEBucket.py
64@@ -33,6 +33,7 @@ from spade.ObjectIO import ObjectIO
65 DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
66 DEFAULT_DELIMITER = "/"
67
68+
69 class GCEBucket():
70 def __init__(self, creds):
71 wanted = ["client_email",
72@@ -77,7 +78,6 @@ class GCEBucket():
73
74 def listBucketContents(self, bucketName, path=None, delimiter=DEFAULT_DELIMITER):
75 contents = []
76- #fields_to_return = 'nextPageToken,prefixes,items(name,size,contentType,timeCreated,updated,md5Hash)'
77 fields_to_return = 'nextPageToken,prefixes,items'
78 req = self.gce.objects().list(bucket=bucketName,
79 prefix=path,
80@@ -146,7 +146,7 @@ class GCEBucket():
81 def getObjectChecksum(self, bucketName, objectName):
82 try:
83 metadata = self.getObjectMetadata(bucketName, objectName)
84- except BucketError as e:
85+ except BucketError:
86 return ""
87 # GCE stores the md5sum in base64, so we need to decode it
88 checksum = base64.b64decode(metadata.rawMetadata.get("md5Hash", ""))
89diff --git a/spade/LocalBucket.py b/spade/LocalBucket.py
90index 12bb517..f2293ec 100644
91--- a/spade/LocalBucket.py
92+++ b/spade/LocalBucket.py
93@@ -9,6 +9,7 @@ from spade.ObjectIO import ObjectIO
94 DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
95 DEFAULT_DELIMITER = "/"
96
97+
98 class LocalBucket():
99 def __init__(self, creds):
100 wanted = ["bucket_root"]
101@@ -87,7 +88,7 @@ class LocalBucket():
102 def getObjectChecksum(self, bucketName, objectName):
103 try:
104 metadata = self.getObjectMetadata(bucketName, objectName)
105- except BucketError as e:
106+ except BucketError:
107 return ""
108 return metadata.rawMetadata.get("etag", "")
109
110@@ -123,7 +124,7 @@ class LocalBucket():
111 try:
112 os.makedirs(pathname)
113 except OSError:
114- raise BucketError("Local Failed to write object - unable to create directory")
115+ raise BucketError("Local Failed to write object - unable to create directory")
116 try:
117 dst = open(fullPath, "wb")
118 except IOError:
119@@ -164,4 +165,3 @@ class LocalBucket():
120
121 def setAcls(self, bucketName, acls, objectName=None):
122 raise NotImplementedError
123-
124diff --git a/spade/ObjectIO.py b/spade/ObjectIO.py
125index 8b969c0..fc2b38f 100644
126--- a/spade/ObjectIO.py
127+++ b/spade/ObjectIO.py
128@@ -1,6 +1,7 @@
129 import os
130 from spade.BucketObject import BucketError
131
132+
133 class ObjectIO():
134 def __init__(self, parent, bucketName, objectName, blockSize=-1):
135 self.parent = parent
136@@ -75,7 +76,6 @@ class ObjectIO():
137 if not line:
138 break
139 yield line
140-
141
142 def write(self, data):
143 raise NotImplementedError
144diff --git a/spade/S3Bucket.py b/spade/S3Bucket.py
145index a846311..90e6a84 100644
146--- a/spade/S3Bucket.py
147+++ b/spade/S3Bucket.py
148@@ -28,6 +28,7 @@ MAX_S3_OBJECTSIZE = 5 * 1024 * 1024 * 1024 # S3 object PUT limit = 5GB
149 S3_SEGMENTSIZE = 1024 * 1024 * 1024 # Use 1GB segments for large object uploads
150 USER_META_PREFIX = "x-amz-meta-"
151
152+
153 class S3Bucket():
154 def __init__(self, creds):
155 wanted = ["aws_access_key_id",
156@@ -43,7 +44,8 @@ class S3Bucket():
157
158 def oldLibraryFixes(self):
159 try:
160- self.regions = [{"name": region.name, "endpoint": region.endpoint} for region in boto.regioninfo.get_regions("s3")]
161+ self.regions = [{"name": region.name, "endpoint": region.endpoint}
162+ for region in boto.regioninfo.get_regions("s3")]
163 except AttributeError:
164 # Old boto library doesn't have this, so we'll use the ones we know about
165 self.regions = [
166@@ -59,7 +61,7 @@ class S3Bucket():
167 {'name': 'us-west-1', 'endpoint': 's3-us-west-1.amazonaws.com'},
168 {'name': 'us-west-2', 'endpoint': 's3-us-west-2.amazonaws.com'},
169 {'name': 'us-east-1', 'endpoint': 's3.amazonaws.com'},
170- ]
171+ ]
172
173 self.base_fields = set(['content-length', 'content-language', 'content-disposition',
174 'content-encoding', 'expires', 'content-md5', 'last-modified',
175@@ -168,7 +170,7 @@ class S3Bucket():
176 def getObjectChecksum(self, bucketName, objectName):
177 try:
178 metadata = self.getObjectMetadata(bucketName, objectName)
179- except BucketError as e:
180+ except BucketError:
181 return ""
182 # Need to strip of the extra quotes around the etag
183 return metadata.rawMetadata.get("etag", "").strip('"')
184@@ -385,7 +387,7 @@ class S3Bucket():
185 for aclType in self.aclTypes:
186 for user in acls.get(aclType):
187 key = user["s3_id"] + user["s3_permission"]
188- if not key in seenPermissions:
189+ if key not in seenPermissions:
190 seenPermissions.append(key)
191 uniqueAcls.append(user)
192
193diff --git a/spade/Spade.py b/spade/Spade.py
194index 5c29617..b33edb0 100644
195--- a/spade/Spade.py
196+++ b/spade/Spade.py
197@@ -5,7 +5,7 @@ import argparse
198 import re
199 import json
200 import fnmatch
201-from spade.BucketObject import BucketObject, BucketError, BucketAcl
202+from spade.BucketObject import BucketObject, BucketError
203 try:
204 from urllib.parse import urlparse
205 except ImportError:
206@@ -28,6 +28,7 @@ else:
207 DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
208 DEFAULT_DELIMITER = "/"
209
210+
211 class Result():
212 def __init__(self):
213 self.output = []
214@@ -36,6 +37,7 @@ class Result():
215 def __str__(self):
216 return "<Result object. output: '{}' error: '{}'>".format(self.output, self.error)
217
218+
219 class Bucket():
220 def __init__(self, configFile):
221 self.configFile = configFile
222@@ -82,12 +84,12 @@ class Bucket():
223 if pattern == prefix:
224 # No wildcards, don't need to list the bucket
225 paths = [pattern]
226- else:
227+ else:
228 # Get a list of all files with the same prefix
229 paths = []
230 try:
231 data = bucket.listBucketContents(url.netloc, path=prefix)
232- except BucketError as e:
233+ except BucketError:
234 return []
235 for obj in data:
236 paths.append(obj.name)
237@@ -95,7 +97,6 @@ class Bucket():
238 # Now naow down the list to the specific glob pattern
239 return fnmatch.filter(paths, pattern)
240
241-
242 def _connect(self, scheme):
243 bucketType = self.creds[scheme]["bucket_type"]
244
245@@ -268,7 +269,7 @@ class Bucket():
246 ret.error = "Invalid bucket type or credentials in config"
247 yield ret
248
249- objList = self._expandGlob(bucket,url)
250+ objList = self._expandGlob(bucket, url)
251 multiple = len(objList) > 1
252
253 for objName in objList:
254@@ -428,7 +429,7 @@ class Bucket():
255 dstObjName = srcObjName
256 elif dstUrl.path.endswith(args.delimiter):
257 # Path provided, so append the source object name (not the full path)
258- dstObjName= dstUrl.path + srcObjName.replace(pseudoDir, "")
259+ dstObjName = dstUrl.path + srcObjName.replace(pseudoDir, "")
260 else:
261 dstObjName = dstUrl.path
262
263diff --git a/spade/StdioBucket.py b/spade/StdioBucket.py
264index 8e110e6..ef6cb43 100644
265--- a/spade/StdioBucket.py
266+++ b/spade/StdioBucket.py
267@@ -1,6 +1,5 @@
268 # Stdin/Stdout "bucket"
269
270-import os
271 import sys
272 import datetime
273 from spade.BucketObject import BucketObject, BucketError
274@@ -8,6 +7,7 @@ from spade.ObjectIO import ObjectIO
275
276 DEFAULT_BLOCKSIZE = 1024 * 1024 * 10
277
278+
279 class StdioBucket():
280 def __init__(self, creds):
281 return
282@@ -29,7 +29,7 @@ class StdioBucket():
283 def getObjectChecksum(self, bucketName, objectName):
284 try:
285 metadata = self.getObjectMetadata(bucketName, objectName)
286- except BucketError as e:
287+ except BucketError:
288 return ""
289 return metadata.rawMetadata.get("etag", "")
290
291@@ -64,5 +64,5 @@ class StdioBucket():
292 # Pretend we did something
293 return
294
295- def deleteBucket(self,bucketName):
296+ def deleteBucket(self, bucketName):
297 raise NotImplementedError
298diff --git a/spade/SwiftBucket.py b/spade/SwiftBucket.py
299index 038d880..f49f2b9 100644
300--- a/spade/SwiftBucket.py
301+++ b/spade/SwiftBucket.py
302@@ -28,6 +28,7 @@ SWIFT_SEGMENTSIZE = 1024 * 1024 * 1024 # Use 1GB segments for large obj
303 USER_META_PREFIX = "x-object-meta-"
304 ACL_META_PREFIX = "x-container-"
305
306+
307 class SwiftBucket():
308 def __init__(self, creds):
309 wanted = ["os_username",
310@@ -157,7 +158,7 @@ class SwiftBucket():
311 def getObjectChecksum(self, bucketName, objectName):
312 try:
313 metadata = self.getObjectMetadata(bucketName, objectName)
314- except BucketError as e:
315+ except BucketError:
316 return ""
317 return metadata.rawMetadata.get("etag", "").strip('"')
318
319@@ -236,7 +237,7 @@ class SwiftBucket():
320 headers = {
321 'x-object-manifest': manifest,
322 'x-object-meta-mtime': mtime,
323- }
324+ }
325 try:
326 self.swift.put_object(bucketName,
327 objectName,
328@@ -267,7 +268,7 @@ class SwiftBucket():
329 def deleteObject(self, bucketName, objectName):
330 try:
331 metadata = self.getObjectMetadata(bucketName, objectName)
332- except BucketError as e:
333+ except BucketError:
334 raise BucketError("Swift Failed to fetch object metadata")
335 manifest = metadata.rawMetadata.get("x-object-manifest", None)
336 if manifest:
337diff --git a/spade/spade.py b/spade/spade.py
338index 45ad465..c0404a2 100755
339--- a/spade/spade.py
340+++ b/spade/spade.py
341@@ -7,6 +7,7 @@ import sys
342 import argparse
343 from spade.Spade import Bucket
344
345+
346 def main():
347 parser = argparse.ArgumentParser(description="Bucket manipulation")
348 parser.add_argument("--configfile", dest="configfile", help="Config file")

Subscribers

People subscribed via source and target branches

to all changes: