Merge lp:~dernils/duplicity/robust-dropbox-backend into lp:~duplicity-team/duplicity/0.8-series

[nils]

Added new command line option --backend-retry-delay that allows to determine the time that duplicity sleeps before retrying after an error has occured.

Added some robustness to dpbxbackend.py that ensures re-authentication happens in case that a socket is changed (e.g. due to a forced reconnect of a dynamic internet connection).

[edso (ed.so)]

hey Nils,

looks good, but please add the new parameter to the man page /bin/duplicity.1 as well. no use to have a parameter that is not properly documented.

thanks.. ede

review: Needs Fixing

[lp:~dernils/duplicity/robust-dropbox-backend]

1157. By nils on 2016-11-17

Added --backend-retry-delay to the manpage

[nils (dernils)]

Good point!
[x] done

[edso (ed.so)]

close, but your missing the parameter eg.

.BI "--backend-retry-delay " number

..ede

review: Needs Fixing

[lp:~dernils/duplicity/robust-dropbox-backend]

1158. By nils on 2016-11-17

minor fix in manpage

[nils (dernils)]

:-)
fixed

[edso (ed.so)]

review: Approve

[Kenneth Loafman (kenneth-loafman)]

This had already been merged. LP did not mark it.

[nils (dernils)]

Thanks for letting me know. I was already a little bit surprised ;-)

Kenneth Loafman <email address hidden> schrieb am Sa., 29. Apr. 2017 um
14:50 Uhr:

> This had already been merged. LP did not mark it.
>
> --
>
> https://code.launchpad.net/~dernils/duplicity/robust-dropbox-backend/+merge/311073
> You are the owner of lp:~dernils/duplicity/robust-dropbox-backend.
>

=== modified file 'bin/duplicity.1'
--- bin/duplicity.1	2016-11-01 19:26:03 +0000
+++ bin/duplicity.1	2016-11-17 11:36:06 +0000
@@ -313,6 +313,12 @@
 enough storage space is required to store two volumes.
 
 .TP
+.BI "--backend-retry-delay " number
+Specifies the number of seconds that duplicity waits after an error has
+occured before attempting to repeat the operation.
+
+
+.TP
 .BI "--cf-backend " backend
 Allows the explicit selection of a cloudfiles backend. Defaults to
 .BR pyrax .

=== modified file 'duplicity/backend.py'
--- duplicity/backend.py	2016-06-28 21:03:46 +0000
+++ duplicity/backend.py	2016-11-17 11:36:06 +0000
@@ -395,9 +395,9 @@
                                  % (n, e.__class__.__name__, util.uexc(e)))
                     if not at_end:
                         if isinstance(e, TemporaryLoadException):
-                            time.sleep(90)  # wait longer before trying again
+                            time.sleep(3*globals.backend_retry_delay)  # wait longer before trying again
                         else:
-                            time.sleep(30)  # wait a bit before trying again
+                            time.sleep(globals.backend_retry_delay)  # wait a bit before trying again
                         if hasattr(self.backend, '_retry_cleanup'):
                             self.backend._retry_cleanup()
 

=== modified file 'duplicity/backends/dpbxbackend.py'
--- duplicity/backends/dpbxbackend.py	2016-06-28 21:03:46 +0000
+++ duplicity/backends/dpbxbackend.py	2016-11-17 11:36:06 +0000
@@ -98,7 +98,14 @@
         self.auth_flow = None
 
         self.login()
-
+    def user_authenticated(self):
+        try:
+            account = self.api_client.users_get_current_account()
+            log.Debug("User authenticated as ,%s" % account)
+            return True
+        except:
+            log.Debug('User not authenticated')
+            return False
     def load_access_token(self):
         return os.environ.get('DPBX_ACCESS_TOKEN', None)
 
@@ -196,10 +203,14 @@
                                    (res_metadata.size, file_size))
 
     def put_file_small(self, source_path, remote_path):
+        if not self.user_authenticated():
+            self.login()
+
         file_size = os.path.getsize(source_path.name)
         f = source_path.open('rb')
         try:
             log.Debug('dpbx,files_upload(%s, [%d bytes])' % (remote_path, file_size))
+
             res_metadata = self.api_client.files_upload(f, remote_path,
                                                         mode=WriteMode.overwrite,
                                                         autorename=False,
@@ -212,6 +223,9 @@
             f.close()
 
     def put_file_chunked(self, source_path, remote_path):
+        if not self.user_authenticated():
+            self.login()
+
         file_size = os.path.getsize(source_path.name)
         f = source_path.open('rb')
         try:
@@ -295,6 +309,10 @@
                     log.Debug('dpbx,files_upload_session_append: %s' % e)
 
                     retry_number -= 1
+
+                    if not self.user_authenticated():
+                        self.login()
+
                     if retry_number == 0:
                         raise
 
@@ -320,6 +338,9 @@
 
     @command()
     def _get(self, remote_filename, local_path):
+        if not self.user_authenticated():
+            self.login()
+
         remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
         remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
 
@@ -353,6 +374,8 @@
     @command()
     def _list(self):
         # Do a long listing to avoid connection reset
+        if not self.user_authenticated():
+            self.login()
         remote_dir = '/' + urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()
 
         log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
@@ -373,6 +396,9 @@
 
     @command()
     def _delete(self, filename):
+        if not self.user_authenticated():
+            self.login()
+
         remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
         remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
 
@@ -390,6 +416,8 @@
 
     @command()
     def _query(self, filename):
+        if not self.user_authenticated():
+            self.login()
         remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
         remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
 
@@ -399,6 +427,8 @@
         return {'size': info.size}
 
     def check_renamed_files(self, file_list):
+        if not self.user_authenticated():
+            self.login()
         bad_list = [x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None]
         if len(bad_list) == 0:
             return

=== modified file 'duplicity/commandline.py'
--- duplicity/commandline.py	2016-03-01 15:38:58 +0000
+++ duplicity/commandline.py	2016-11-17 11:36:06 +0000
@@ -624,6 +624,11 @@
                       metavar=_("path"), dest="file_changed",
                       callback=lambda o, s, v, p: setattr(p.values, "file_changed", v.rstrip('/')))
 
+    # delay time before next try after a failure of a backend operation
+    # TRANSL: Used in usage help. Example:
+    # --backend-retry-delay <seconds>
+    parser.add_option("--backend-retry-delay", type="int", metavar=_("seconds"))
+                      
     # parse the options
     (options, args) = parser.parse_args()
 

=== modified file 'duplicity/globals.py'
--- duplicity/globals.py	2016-06-12 13:13:57 +0000
+++ duplicity/globals.py	2016-11-17 11:36:06 +0000
@@ -293,3 +293,6 @@
 
 # If set, collect only the file status, not the whole root.
 file_changed = None
+
+# delay (in seconds) before next operation after failure
+backend_retry_delay = 30