Merge ~barryprice/container-log-archive-charm/+git/container-log-archive-charm:master into container-log-archive-charm:master

[Barry Price]

Add retry logic to swift uploads, cRT#129788

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Tom Haddon (mthaddon)]

One minor comment inline

[Tom Haddon (mthaddon)]

LGTM, thx

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 141e8604f9ec6cbcad433f43a8b108d32196bd7b

diff --git a/files/archive-to-swift.py b/files/archive-to-swift.py
index 535c7f5..1d480b4 100755
--- a/files/archive-to-swift.py
+++ b/files/archive-to-swift.py
@@ -114,13 +114,20 @@ class SwiftCloudStorage(CloudStorage):
                 segment_name = '{}/{}/{}/{}/{:08d}'.format(file_name, mtime, file_size, self.segment_size, segment_id)
                 segment_id += 1
                 log.info("Uploading segment {}".format(segment_id))
-                try:
-                    self.swift.put_object(segment_bucket,
-                                          self.expand_path(segment_name),
-                                          data,
-                                          content_type="application/octet-stream")
-                except swiftclient.ClientException:
-                    log.info("Segment upload failed!")
+                max_retries = 10
+                for attempt in range(max_retries):
+                    try:
+                        self.swift.put_object(segment_bucket,
+                                              self.expand_path(segment_name),
+                                              data,
+                                              content_type="application/octet-stream")
+                    except swiftclient.ClientException:
+                        log.info("Segment upload failed on attempt {} - retrying".format(attempt))
+                    else:
+                        log.info("Segment upload succeeded on attempt {}".format(attempt))
+                        break
+                else:
+                    log.info("Segment upload failed too many times ({}) - aborting upload".format(attempt))
                     return Status.FAILURE
 
         # New we have all the part uploaded, create the manifest
@@ -156,18 +163,24 @@ class SwiftCloudStorage(CloudStorage):
 
         try:
             with open(path, "rb") as obj:
-                try:
-                    self.swift.put_object(self.bucket,
-                                          self.expand_path(file_name),
-                                          obj,
-                                          content_type="application/octet-stream")
-                except swiftclient.ClientException:
-                    log.info("Upload failed")
+                max_retries = 10
+                for attempt in range(max_retries):
+                    try:
+                        self.swift.put_object(self.bucket,
+                                              self.expand_path(file_name),
+                                              obj,
+                                              content_type="application/octet-stream")
+                    except swiftclient.ClientException:
+                        log.info("Upload failed on attempt {} - retrying".format(attempt))
+                    else:
+                        log.info("Upload succeeded on attempt {}".format(attempt))
+                        return Status.SUCCESS
+                else:
+                    log.info("Upload failed too many times ({}) - aborting upload".format(attempt))
                     return Status.FAILURE
         except IOError:
             log.info("Unable to read {}".format(path))
             return Status.FAILURE
-        return Status.SUCCESS
 
     def delete(self, name):
         """ Delete an object from cloud storage