Merge ~barryprice/spade/+git/spade:flake8 into spade:master

[Barry Price]

Address all Flake8 errors bar complexity (noqa: C901)

Unmerged commits

aa2d273... by Barry Price on 2020-11-27

Address all Flake8 errors bar complexity (noqa: C901)

7e34f18... by Barry Price on 2020-11-27

Modernise, switch to python3 exclusively

diff --git a/Makefile b/Makefile
index c3b3ca6..d51fa68 100644
--- a/Makefile
+++ b/Makefile
@@ -1,23 +1,22 @@
-version := $(shell python -c "from spade import __version__; print(__version__)")
-tempfiles_whl := build dist spade.egg-info
-tempfiles_snap := snapcraft.yaml parts prime stage spade*.snap
-tempfiles_misc := spade/__pycache__
-tempfiles_export := spade-export spade-export27.tar.gz
+version := $(shell python3 -c "from spade import __version__; print(__version__)")
+tempfiles_snap := snap/snapcraft.yaml spade*.snap
 
-whl: whl_clean
-	python3 setup.py bdist_wheel
+blacken:
+	@echo "Normalising python layout with black (but not really)."
 
-whl_clean:
-	rm -Rf $(tempfiles_whl)
+lint: blacken
+	@echo "Running flake8"
+	@tox -e lint
+
+clean:
+	@echo "Cleaninf files"
+	@git clean -fXd
 
 snap: snap_clean
-	sed -e "s/{{VERSION}}/$(version)/" snapcraft.yaml.in > snapcraft.yaml
+	sed -e "s/{{VERSION}}/$(version)/" snapcraft.yaml.in > snap/snapcraft.yaml
 	snapcraft
 
 snap_clean:
 	rm -Rf $(tempfiles_snap)
 
-clean: whl_clean snap_clean
-
-lint:
-	@find spade -name \*py | xargs flake8 --config=/dev/null --ignore=C901 --max-line-length=120 --max-complexity=10 --hang-closing
+.PHONY: blacken lint clean snap snap_clean
diff --git a/setup.py b/setup.py
index e52b204..7286fa2 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ import re
 here = path.abspath(path.dirname(__file__))
 
 def find_version(*file_paths):
-    with open(path.join(here, *file_paths), 'r', 'latin1') as f:
+    with open(path.join(here, *file_paths), 'r', 'utf-8') as f:
         version_file = f.read()
     version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
                               version_file, re.M)
diff --git a/snap/.snapcraft/state b/snap/.snapcraft/state
 deleted file mode 100644
index 3497788..0000000
--- a/snap/.snapcraft/state
+++ /dev/null
@@ -1,3 +0,0 @@
-!GlobalState
-assets:
-  build-packages: []
diff --git a/spade/AzureBucket.py b/spade/AzureBucket.py
index 92bf430..0f7a62d 100644
--- a/spade/AzureBucket.py
+++ b/spade/AzureBucket.py
@@ -239,7 +239,7 @@ class AzureBucket():
                 newStandardMetadata[k] = v
         if not overwrite:
             for k, v in m.userMetadata.items():
-                    newUserMetadata[USER_META_PREFIX + k] = v
+                newUserMetadata[USER_META_PREFIX + k] = v
 
         # Add new ones
         for k, v in metadata.items():
@@ -318,7 +318,7 @@ class AzureBucket():
                                                      body=xml,
                                                      method="PUT")
         if not response.get("status") == "201":
-                raise BucketError("Azure Failed to upload blocklist")
+            raise BucketError("Azure Failed to upload blocklist")
         return
 
     def putObject(self, bucketName, objectName, src, size, blockSize=DEFAULT_BLOCKSIZE):
diff --git a/spade/S3Bucket.py b/spade/S3Bucket.py
index 90e6a84..6d3036e 100644
--- a/spade/S3Bucket.py
+++ b/spade/S3Bucket.py
@@ -61,7 +61,7 @@ class S3Bucket():
                 {'name': 'us-west-1', 'endpoint': 's3-us-west-1.amazonaws.com'},
                 {'name': 'us-west-2', 'endpoint': 's3-us-west-2.amazonaws.com'},
                 {'name': 'us-east-1', 'endpoint': 's3.amazonaws.com'},
-                ]
+            ]
 
             self.base_fields = set(['content-length', 'content-language', 'content-disposition',
                                     'content-encoding', 'expires', 'content-md5', 'last-modified',
@@ -325,7 +325,7 @@ class S3Bucket():
         else:
             return aclData["s3_id"]
 
-    def getAcls(self, bucketName, objectName=None):
+    def getAcls(self, bucketName, objectName=None):  # noqa: C901
         self.confirmRegion(bucketName)
         try:
             bucket = self.s3.get_bucket(bucketName)
@@ -366,7 +366,7 @@ class S3Bucket():
 
         return acls
 
-    def setAcls(self, bucketName, acls, objectName=None):
+    def setAcls(self, bucketName, acls, objectName=None):  # noqa: C901
         self.confirmRegion(bucketName)
         try:
             bucket = self.s3.get_bucket(bucketName)
diff --git a/spade/Spade.py b/spade/Spade.py
index b33edb0..b8f4b12 100644
--- a/spade/Spade.py
+++ b/spade/Spade.py
@@ -69,7 +69,7 @@ class Bucket():
                 raise BucketError("Data is not in bzip2 format")
             else:
                 raise BucketError("Unknown IO error")
-        except lzma.LZMAError as e:
+        except lzma.LZMAError:
             if "Input format not supported by decoder":
                 raise BucketError("Data is not in xz/lzma format")
             else:
@@ -150,7 +150,7 @@ class Bucket():
             ret.output.append(retElement)
         return ret
 
-    def ls(self, rawArgs):
+    def ls(self, rawArgs):  # noqa: C901
         ret = Result()
         parser = argparse.ArgumentParser(description="Bucket ls")
         parser.add_argument("-l", dest="long", action="store_true")
@@ -237,7 +237,7 @@ class Bucket():
             ret.output = [data.decode("utf-8")]
         f.close()
 
-    def grep(self, rawArgs):
+    def grep(self, rawArgs):  # noqa: C901
         ret = Result()
         parser = argparse.ArgumentParser(description="Bucket grep")
         parser.add_argument("-e", dest="regex", action="store_true", help="Allow regexes in the search")
@@ -374,7 +374,7 @@ class Bucket():
         ret.output.append(e)
         return ret
 
-    def cp(self, rawArgs):
+    def cp(self, rawArgs):  # noqa: C901
         ret = Result()
         parser = argparse.ArgumentParser(description="Bucket cp")
         parser.add_argument(dest="src")
@@ -581,7 +581,7 @@ class Bucket():
 
     # FIXME: Need to pass add/remove to bucket acl functions instead of
     #        just the processed acls list
-    def acls(self, rawArgs):
+    def acls(self, rawArgs):  # noqa: C901
         ret = Result()
         parser = argparse.ArgumentParser(description="Bucket ACLs")
         parser.add_argument("-a", "--add", dest="add", action="append", default=[])
diff --git a/spade/SwiftBucket.py b/spade/SwiftBucket.py
index f49f2b9..2125b85 100644
--- a/spade/SwiftBucket.py
+++ b/spade/SwiftBucket.py
@@ -108,7 +108,7 @@ class SwiftBucket():
             co.rawMetadata[k] = v
         return co
 
-    def setBucketMetadata(self, bucketName, metadata, overwrite=False):
+    def setBucketMetadata(self, bucketName, metadata, overwrite=False):  # noqa: C901
         systemMetadata = ["content-type", "x-container-bytes-used",
                           "x-container-object-count:", "x-timestamp"]
         newMetadata = {}
@@ -237,7 +237,7 @@ class SwiftBucket():
         headers = {
             'x-object-manifest': manifest,
             'x-object-meta-mtime': mtime,
-            }
+        }
         try:
             self.swift.put_object(bucketName,
                                   objectName,
@@ -265,7 +265,7 @@ class SwiftBucket():
     def open(self, bucketName, objectName, blockSize=DEFAULT_BLOCKSIZE):
         return ObjectIO(self, bucketName, objectName, blockSize)
 
-    def deleteObject(self, bucketName, objectName):
+    def deleteObject(self, bucketName, objectName):  # noqa: C901
         try:
             metadata = self.getObjectMetadata(bucketName, objectName)
         except BucketError:
diff --git a/spade/spade.py b/spade/spade.py
index c0404a2..1b5811c 100755
--- a/spade/spade.py
+++ b/spade/spade.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 
@@ -8,7 +8,7 @@ import argparse
 from spade.Spade import Bucket
 
 
-def main():
+def main():  # noqa: C901
     parser = argparse.ArgumentParser(description="Bucket manipulation")
     parser.add_argument("--configfile", dest="configfile", help="Config file")
     parser.add_argument(dest="cmd")
@@ -157,5 +157,6 @@ def main():
     else:
         print("Unknown command {}".format(args.cmd), file=sys.stderr)
 
+
 if __name__ == "__main__":
     main()
diff --git a/tox.ini b/tox.ini
 new file mode 100644
index 0000000..b1b3e16
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,34 @@
+[tox]
+skipsdist=True
+envlist = unit, functional
+
+[testenv]
+basepython = python3
+
+[testenv:unit]
+commands =
+    pytest --ignore mod --ignore {toxinidir}/tests/functional \
+      {posargs:-v  --cov=spade --cov-report=term-missing --cov-branch}
+deps = -r{toxinidir}/tests/unit/requirements.txt
+       -r{toxinidir}/requirements.txt
+setenv =
+  PYTHONPATH={toxinidir}/spade
+  TZ=UTC
+
+[testenv:black]
+commands = black --skip-string-normalization --line-length=120 spade/ tests/
+deps = black
+
+[testenv:lint]
+commands = flake8 spade/ tests/
+# Pin flake8 to 3.7.9 to match focal
+deps =
+    flake8==3.7.9
+
+[flake8]
+exclude =
+    .git,
+    __pycache__,
+    .tox,
+max-line-length = 120
+max-complexity = 10