Merge lp:~ed.so/duplicity/webdav.fix-retry into lp:duplicity/0.6

[edso]

webdav
- added ssl certificate verification (see man page)
- more robust retry routine to survive ssl errors, broken pipe errors
- added http redirect support

webdav
- added ssl certificate verification (see man page)
- more robust retry routine to survive ssl errors, broken pipe errors
- added http redirect support

[Kenneth Loafman (kenneth-loafman)]

The 'Diff against target' has conflicts (see above). Needs to be fixed.

[edso (ed.so)]

would simply merging interim manpage changes in this branch solve this?

..ede

[Kenneth Loafman (kenneth-loafman)]

Possibly, bzr is fairly fussy about merges.

...Ken

On Fri, Jan 11, 2013 at 4:51 AM, edso <email address hidden> wrote:

> would simply merging interim manpage changes in this branch solve this?
>
> ..ede
> --
> https://code.launchpad.net/~ed.so/duplicity/webdav.fix-retry/+merge/142759
> You are subscribed to branch lp:duplicity.
>

[lp:~ed.so/duplicity/webdav.fix-retry]

904. By ede on 2013-01-11

move man page changes to a separate uptodate merge branch

[edso (ed.so)]

merge problem should be removed
please merge manpage branch as well

review: Approve

=== modified file 'duplicity/backend.py'
--- duplicity/backend.py	2012-12-12 17:45:38 +0000
+++ duplicity/backend.py	2013-01-11 18:46:21 +0000
@@ -42,7 +42,7 @@
 
 from duplicity.util import exception_traceback
 
-from duplicity.errors import BackendException
+from duplicity.errors import BackendException, FatalBackendError
 from duplicity.errors import TemporaryLoadException
 from duplicity.errors import ConflictingScheme
 from duplicity.errors import InvalidBackendURL
@@ -333,26 +333,35 @@
 # as we don't know what the underlying code comes up with and we really *do*
 # want to retry globals.num_retries times under all circumstances
 def retry_fatal(fn):
-    def iterate(*args):
-        for n in range(1, globals.num_retries):
-            try:
-                return fn(*args)
-            except Exception, e:
-                log.Warn("Attempt %s failed. %s: %s"
-                         % (n, e.__class__.__name__, str(e)))
-                log.Debug("Backtrace of previous error: %s"
-                          % exception_traceback())
-                time.sleep(10) # wait a bit before trying again
+    def _retry_fatal(self, *args):
+        try:
+            n = 0
+            for n in range(1, globals.num_retries):
+                try:
+                    self.retry_count = n
+                    return fn(self, *args)
+                except FatalBackendError, e:
+                    # die on fatal errors
+                    raise e
+                except Exception, e:
+                    # retry on anything else
+                    log.Warn("Attempt %s failed. %s: %s"
+                             % (n, e.__class__.__name__, str(e)))
+                    log.Debug("Backtrace of previous error: %s"
+                              % exception_traceback())
+                    time.sleep(10) # wait a bit before trying again
         # final trial, die on exception
-        try:
-            return fn(*args)
+            self.retry_count = n+1
+            return fn(self, *args)
         except Exception, e:
             log.FatalError("Giving up after %s attempts. %s: %s"
-                         % (globals.num_retries, e.__class__.__name__, str(e)),
+                         % (self.retry_count, e.__class__.__name__, str(e)),
                           log.ErrorCode.backend_error)
             log.Debug("Backtrace of previous error: %s"
                         % exception_traceback())
-    return iterate
+        self.retry_count = 0
+
+    return _retry_fatal
 
 class Backend:
     """
@@ -371,6 +380,7 @@
 
       - move
     """
+    
     def __init__(self, parsed_url):
         self.parsed_url = parsed_url
 

=== modified file 'duplicity/backends/webdavbackend.py'
--- duplicity/backends/webdavbackend.py	2012-12-09 14:36:19 +0000
+++ duplicity/backends/webdavbackend.py	2013-01-11 18:46:21 +0000
@@ -2,6 +2,8 @@
 #
 # Copyright 2002 Ben Escoto <[e-mail]>
 # Copyright 2007 Kenneth Loafman <[e-mail]>
+# Copyright 2013 Edgar Soldin
+#                 - ssl cert verification, some robustness enhancements
 #
 # This file is part of duplicity.
 #
@@ -20,7 +22,7 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
 import base64
-import httplib
+import httplib, os
 import re
 import urllib
 import urllib2
@@ -46,6 +48,58 @@
     def get_method(self):
         return self.method
 
+class VerifiedHTTPSConnection(httplib.HTTPSConnection):
+        def __init__(self, *args, **kwargs):
+            try:
+                global socket, ssl
+                import socket, ssl
+            except ImportError:
+                raise FatalBackendError("Missing socket or ssl libraries.")
+            
+            httplib.HTTPSConnection.__init__(self, *args, **kwargs)
+            
+            self.cacert_file = globals.ssl_cacert_file
+            cacert_candidates = [ "~/.duplicity/cacert.pem", \
+                             "~/duplicity_cacert.pem", \
+                             "/etc/duplicity/cacert.pem" ]
+            # 
+            if not self.cacert_file:
+                for path in cacert_candidates :
+                    path = os.path.expanduser(path)
+                    if (os.path.isfile(path)):
+                        self.cacert_file = path
+                        break
+            # still no cacert file, inform user
+            if not self.cacert_file:
+                raise FatalBackendError("""For certificate verification a cacert database file is needed in one of these locations: %s
+Hints: 
+  Consult the man page, chapter 'SSL Certificate Verification'. 
+  Consider using the options --ssl-cacert-file, --ssl-no-check-certificate .""" % ", ".join(cacert_candidates) )
+            # check if file is accessible (libssl errors are not very detailed)
+            if not os.access(self.cacert_file, os.R_OK):
+                raise FatalBackendError("Cacert database file '%s' is not readable." % cacert_file)
+
+        def connect(self):
+            # create new socket
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+
+            # wrap the socket in ssl using verification
+            self.sock = ssl.wrap_socket(sock,
+                                        cert_reqs=ssl.CERT_REQUIRED,
+                                        ca_certs=self.cacert_file,
+                                        )
+        
+        def request(self, *args, **kwargs):
+            try:
+                return httplib.HTTPSConnection.request(self, *args, **kwargs)
+            except ssl.SSLError, e:
+                # encapsulate ssl errors
+                raise BackendException("SSL failed: %s" % str(e),log.ErrorCode.backend_error)
+
 
 class WebDAVBackend(duplicity.backend.Backend):
     """Backend for accessing a WebDAV repository.
@@ -70,23 +124,22 @@
         self.digest_challenge = None
         self.digest_auth_handler = None
 
-        if parsed_url.path:
+        self.username = parsed_url.username
+        self.password = self.get_password()
+        self.directory = self._sanitize_path(parsed_url.path)
+
+        log.Info("Using WebDAV protocol %s" % (globals.webdav_proto,))
+        log.Info("Using WebDAV host %s port %s" % (parsed_url.hostname, parsed_url.port))
+        log.Info("Using WebDAV directory %s" % (self.directory,))
+
+        self.conn = None
+
+    def _sanitize_path(self,path):
+        if path:
             foldpath = re.compile('/+')
-            self.directory = foldpath.sub('/', parsed_url.path + '/' )
-        else:
-            self.directory = '/'
-
-        log.Info("Using WebDAV host %s" % (parsed_url.hostname,))
-        log.Info("Using WebDAV port %s" % (parsed_url.port,))
-        log.Info("Using WebDAV directory %s" % (self.directory,))
-        log.Info("Using WebDAV protocol %s" % (globals.webdav_proto,))
-
-        if parsed_url.scheme == 'webdav':
-            self.conn = httplib.HTTPConnection(parsed_url.hostname, parsed_url.port)
-        elif parsed_url.scheme == 'webdavs':
-            self.conn = httplib.HTTPSConnection(parsed_url.hostname, parsed_url.port)
-        else:
-            raise BackendException("Unknown URI scheme: %s" % (parsed_url.scheme))
+            return foldpath.sub('/', path + '/' )
+        else:
+            return '/'
 
     def _getText(self,nodelist):
         rc = ""
@@ -94,29 +147,76 @@
             if node.nodeType == node.TEXT_NODE:
                 rc = rc + node.data
         return rc
+    
+    def _connect(self, forced=False):
+        """
+        Connect or re-connect to the server, updates self.conn
+        # reconnect on errors as a precaution, there are errors e.g. 
+        # "[Errno 32] Broken pipe" or SSl errors that render the connection unusable
+        """
+        if self.retry_count<=1 and self.conn \
+            and self.conn.host == self.parsed_url.hostname: return
+        
+        log.Info("WebDAV create connection on '%s' (retry %s) " % (self.parsed_url.hostname,self.retry_count) )
+        if self.conn: self.conn.close()
+        # http schemes needed for redirect urls from servers
+        if self.parsed_url.scheme in ['webdav','http']:
+            self.conn = httplib.HTTPConnection(self.parsed_url.hostname, self.parsed_url.port)
+        elif self.parsed_url.scheme in ['webdavs','https']:
+            if globals.ssl_no_check_certificate:
+                self.conn = httplib.HTTPSConnection(self.parsed_url.hostname, self.parsed_url.port)
+            else:
+                self.conn = VerifiedHTTPSConnection(self.parsed_url.hostname, self.parsed_url.port)
+        else:
+            raise FatalBackendError("WebDAV Unknown URI scheme: %s" % (self.parsed_url.scheme))
 
     def close(self):
         self.conn.close()
 
-    def request(self, method, path, data=None):
+    def request(self, method, path, data=None, redirected=0):
         """
         Wraps the connection.request method to retry once if authentication is
         required
         """
-        quoted_path = urllib.quote(path)
+        self._connect()
+
+        quoted_path = urllib.quote(path,"/:~")
 
         if self.digest_challenge is not None:
             self.headers['Authorization'] = self.get_digest_authorization(path)
+
+        log.Info("WebDAV %s %s request with headers: %s " % (method,quoted_path,self.headers))
+        log.Info("WebDAV data length: %s " % len(str(data)) )
         self.conn.request(method, quoted_path, data, self.headers)
         response = self.conn.getresponse()
-        if response.status == 401:
+        log.Info("WebDAV response status %s with reason '%s'." % (response.status,response.reason))
+        # resolve redirects and reset url on listing requests (they usually come before everything else)
+        if response.status in [301,302] and method == 'PROPFIND':
+            redirect_url = response.getheader('location',None)
+            response.close()
+            if redirect_url:
+                log.Notice("WebDAV redirect to: %s " % urllib.unquote(redirect_url) )
+                if redirected > 10:
+                    raise FatalBackendError("WebDAV redirected 10 times. Giving up.")
+                self.parsed_url = duplicity.backend.ParsedUrl(redirect_url)
+                self.directory = self._sanitize_path(self.parsed_url.path)
+                return self.request(method,self.directory,data,redirected+1)
+            else:
+                raise FatalBackendError("WebDAV missing location header in redirect response.")
+        elif response.status == 401:
             response.close()
             self.headers['Authorization'] = self.get_authorization(response, quoted_path)
+            log.Info("WebDAV retry request with authentification headers.")
+            log.Info("WebDAV %s %s request2 with headers: %s " % (method,quoted_path,self.headers))
+            log.Info("WebDAV data length: %s " % len(str(data)) )
             self.conn.request(method, quoted_path, data, self.headers)
             response = self.conn.getresponse()
-
+            log.Info("WebDAV response2 status %s with reason '%s'." % (response.status,response.reason))
+        
         return response
 
+
+
     def get_authorization(self, response, path):
         """
         Fetches the auth header based on the requested method (basic or digest)
@@ -139,7 +239,7 @@
         """
         Returns the basic auth header
         """
-        auth_string = '%s:%s' % (self.parsed_url.username, self.get_password())
+        auth_string = '%s:%s' % (self.username, self.password)
         return 'Basic %s' % base64.encodestring(auth_string).strip()
 
     def get_digest_authorization(self, path):
@@ -149,7 +249,7 @@
         u = self.parsed_url
         if self.digest_auth_handler is None:
             pw_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
-            pw_manager.add_password(None, self.conn.host, u.username, self.get_password())
+            pw_manager.add_password(None, self.conn.host, self.username, self.password)
             self.digest_auth_handler = urllib2.HTTPDigestAuthHandler(pw_manager)
 
         # building a dummy request that gets never sent,
@@ -165,6 +265,7 @@
     def list(self):
         """List files in directory"""
         log.Info("Listing directory %s on WebDAV server" % (self.directory,))
+        response = None
         try:
             self.headers['Depth'] = "1"
             response = self.request("PROPFIND", self.directory, self.listbody)
@@ -178,7 +279,7 @@
                 response.close()
                 # just created folder is so return empty
                 return []
-            elif response.status == 207:
+            elif response.status in [200, 207]:
                 document = response.read()
                 response.close()
             else:
@@ -195,9 +296,10 @@
                 if filename:
                     result.append(filename)
             return result
-        except Exception, cause:
-            e = BackendException("Listing directory %s on WebDAV server failed. %s" % (self.directory,cause))
+        except Exception, e:
             raise e
+        finally:
+            if response: response.close()
 
     def __taste_href(self, href):
         """
@@ -241,11 +343,15 @@
         """Get remote filename, saving it to local_path"""
         url = self.directory + remote_filename
         log.Info("Retrieving %s from WebDAV server" % (url ,))
+        response = None
         try:
             target_file = local_path.open("wb")
             response = self.request("GET", url)
             if response.status == 200:
+                #data=response.read()
                 target_file.write(response.read())
+                #import hashlib
+                #log.Info("WebDAV GOT %s bytes with md5=%s" % (len(data),hashlib.md5(data).hexdigest()) )
                 assert not target_file.close()
                 local_path.setdata()
                 response.close()
@@ -254,9 +360,10 @@
                 reason = response.reason
                 response.close()
                 raise BackendException("Bad status code %s reason %s." % (status,reason))
-        except Exception, cause:
-            e = BackendException("Getting %s from WebDAV server failed. %s" % (url,cause))
+        except Exception, e:
             raise e
+        finally:
+            if response: response.close()
 
     @retry_fatal
     def put(self, source_path, remote_filename = None):
@@ -265,10 +372,11 @@
             remote_filename = source_path.get_filename()
         url = self.directory + remote_filename
         log.Info("Saving %s on WebDAV server" % (url ,))
+        response = None
         try:
             source_file = source_path.open("rb")
             response = self.request("PUT", url, source_file.read())
-            if response.status == 201:
+            if response.status in [201, 204]:
                 response.read()
                 response.close()
             else:
@@ -276,9 +384,10 @@
                 reason = response.reason
                 response.close()
                 raise BackendException("Bad status code %s reason %s." % (status,reason))
-        except Exception, cause:
-            e = BackendException("Putting %s on WebDAV server failed. %s" % (url,cause))
+        except Exception, e:
             raise e
+        finally:
+            if response: response.close()
 
     @retry_fatal
     def delete(self, filename_list):
@@ -286,6 +395,7 @@
         for filename in filename_list:
             url = self.directory + filename
             log.Info("Deleting %s from WebDAV server" % (url ,))
+            response = None
             try:
                 response = self.request("DELETE", url)
                 if response.status == 204:
@@ -296,9 +406,10 @@
                     reason = response.reason
                     response.close()
                     raise BackendException("Bad status code %s reason %s." % (status,reason))
-            except Exception, cause:
-                e = BackendException("Deleting %s on WebDAV server failed. %s" % (url,cause))
+            except Exception, e:
                 raise e
+            finally:
+                if response: response.close()
 
 duplicity.backend.register_backend("webdav", WebDAVBackend)
 duplicity.backend.register_backend("webdavs", WebDAVBackend)

=== modified file 'duplicity/commandline.py'
--- duplicity/commandline.py	2012-11-21 01:27:35 +0000
+++ duplicity/commandline.py	2013-01-11 18:46:21 +0000
@@ -502,6 +502,11 @@
     # user added ssh options
     parser.add_option("--ssh-options", action="extend", metavar=_("options"))
 
+    # user added ssl options (webdav backend)
+    parser.add_option("--ssl-cacert-file", metavar=_("pem formatted bundle of certificate authorities"))
+
+    parser.add_option("--ssl-no-check-certificate", action="store_true")
+
     # Working directory for the tempfile module. Defaults to /tmp on most systems.
     parser.add_option("--tempdir", dest="temproot", type="file", metavar=_("path"))
 

=== modified file 'duplicity/errors.py'
--- duplicity/errors.py	2011-08-03 19:47:29 +0000
+++ duplicity/errors.py	2013-01-11 18:46:21 +0000
@@ -70,6 +70,12 @@
     """
     pass
 
+class FatalBackendError(DuplicityError):
+    """
+    Raised to indicate a backend failed fatally.
+    """
+    pass
+
 class TemporaryLoadException(BackendException):
     """
     Raised to indicate a temporary issue on the backend.

=== modified file 'duplicity/globals.py'
--- duplicity/globals.py	2012-03-13 20:53:35 +0000
+++ duplicity/globals.py	2013-01-11 18:46:21 +0000
@@ -214,6 +214,10 @@
 # whether to use scp for put/get, sftp is default
 use_scp = False
 
+# HTTPS ssl optons (currently only webdav)
+ssl_cacert_file = None
+ssl_no_check_certificate = False
+
 # user added rsync options
 rsync_options = ""