Merge lp:~stapelberg+ubuntu/duplicity/add-onedrive-backend into lp:~duplicity-team/duplicity/0.7-series
- add-onedrive-backend
- Merge into 0.7-series
Proposed by
Michael Stapelberg
Status: | Merged |
---|---|
Merged at revision: | 1044 |
Proposed branch: | lp:~stapelberg+ubuntu/duplicity/add-onedrive-backend |
Merge into: | lp:~duplicity-team/duplicity/0.7-series |
Diff against target: |
376 lines (+344/-0) 3 files modified
bin/duplicity.1 (+13/-0) duplicity/backends/onedrivebackend.py (+330/-0) duplicity/commandline.py (+1/-0) |
To merge this branch: | bzr merge lp:~stapelberg+ubuntu/duplicity/add-onedrive-backend |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
duplicity-team | Pending | ||
Review via email: mp+245534@code.launchpad.net |
Commit message
Add a Microsoft OneDrive backend
Description of the change
Add a Microsoft OneDrive backend
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'bin/duplicity.1' | |||
2 | --- bin/duplicity.1 2014-12-10 19:09:28 +0000 | |||
3 | +++ bin/duplicity.1 2015-01-04 21:11:21 +0000 | |||
4 | @@ -91,6 +91,13 @@ | |||
5 | 91 | .B Python library for mega API | 91 | .B Python library for mega API |
6 | 92 | - https://github.com/ckornacker/mega.py, ubuntu ppa - ppa:ckornacker/backup | 92 | - https://github.com/ckornacker/mega.py, ubuntu ppa - ppa:ckornacker/backup |
7 | 93 | .TP | 93 | .TP |
8 | 94 | .BR "OneDrive backend" " (Microsoft OneDrive)" | ||
9 | 95 | .B python-requests | ||
10 | 96 | - http://python-requests.org | ||
11 | 97 | .br | ||
12 | 98 | .B python-requests-oauthlib | ||
13 | 99 | - https://github.com/requests/requests-oauthlib | ||
14 | 100 | .TP | ||
15 | 94 | .BR "ncftp backend" " (ftp, select via ncftp+ftp://)" | 101 | .BR "ncftp backend" " (ftp, select via ncftp+ftp://)" |
16 | 95 | .B NcFTP | 102 | .B NcFTP |
17 | 96 | - http://www.ncftp.com/ | 103 | - http://www.ncftp.com/ |
18 | @@ -1178,6 +1185,12 @@ | |||
19 | 1178 | mega://user[:password]@mega.co.nz/some_dir | 1185 | mega://user[:password]@mega.co.nz/some_dir |
20 | 1179 | .RE | 1186 | .RE |
21 | 1180 | .PP | 1187 | .PP |
22 | 1188 | .B "OneDrive Backend" | ||
23 | 1189 | .PP | ||
24 | 1190 | .RS | ||
25 | 1191 | onedrive://some_dir | ||
26 | 1192 | .RE | ||
27 | 1193 | .PP | ||
28 | 1181 | .B "Par2 Wrapper Backend" | 1194 | .B "Par2 Wrapper Backend" |
29 | 1182 | .PP | 1195 | .PP |
30 | 1183 | .RS | 1196 | .RS |
31 | 1184 | 1197 | ||
32 | === added file 'duplicity/backends/onedrivebackend.py' | |||
33 | --- duplicity/backends/onedrivebackend.py 1970-01-01 00:00:00 +0000 | |||
34 | +++ duplicity/backends/onedrivebackend.py 2015-01-04 21:11:21 +0000 | |||
35 | @@ -0,0 +1,330 @@ | |||
36 | 1 | # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- | ||
37 | 2 | # vim:tabstop=4:shiftwidth=4:expandtab | ||
38 | 3 | # | ||
39 | 4 | # Copyright 2014 Google Inc. | ||
40 | 5 | # Contact Michael Stapelberg <stapelberg+duplicity@google.com> | ||
41 | 6 | # This is NOT a Google product. | ||
42 | 7 | # | ||
43 | 8 | # This file is part of duplicity. | ||
44 | 9 | # | ||
45 | 10 | # Duplicity is free software; you can redistribute it and/or modify it | ||
46 | 11 | # under the terms of the GNU General Public License as published by the | ||
47 | 12 | # Free Software Foundation; either version 2 of the License, or (at your | ||
48 | 13 | # option) any later version. | ||
49 | 14 | # | ||
50 | 15 | # Duplicity is distributed in the hope that it will be useful, but | ||
51 | 16 | # WITHOUT ANY WARRANTY; without even the implied warranty of | ||
52 | 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
53 | 18 | # General Public License for more details. | ||
54 | 19 | # | ||
55 | 20 | # You should have received a copy of the GNU General Public License | ||
56 | 21 | # along with duplicity; if not, write to the Free Software Foundation, | ||
57 | 22 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
58 | 23 | |||
59 | 24 | import time | ||
60 | 25 | import json | ||
61 | 26 | import os | ||
62 | 27 | import sys | ||
63 | 28 | # On debian (and derivatives), get these dependencies using: | ||
64 | 29 | # apt-get install python-requests python-requests-oauthlib | ||
65 | 30 | # On fedora (and derivatives), get these dependencies using: | ||
66 | 31 | # yum install python-requests python-requests-oauthlib | ||
67 | 32 | import requests | ||
68 | 33 | from requests_oauthlib import OAuth2Session | ||
69 | 34 | |||
70 | 35 | import duplicity.backend | ||
71 | 36 | from duplicity.errors import BackendException | ||
72 | 37 | from duplicity import globals | ||
73 | 38 | from duplicity import log | ||
74 | 39 | |||
75 | 40 | # For documentation on the API, see | ||
76 | 41 | # http://msdn.microsoft.com/en-us/library/dn659752.aspx | ||
77 | 42 | # http://msdn.microsoft.com/en-us/library/dn631844.aspx | ||
78 | 43 | # https://gist.github.com/rgregg/37ba8929768a62131e85 | ||
79 | 44 | class OneDriveBackend(duplicity.backend.Backend): | ||
80 | 45 | """Uses Microsoft OneDrive (formerly SkyDrive) for backups.""" | ||
81 | 46 | |||
82 | 47 | OAUTH_TOKEN_PATH = os.path.expanduser( | ||
83 | 48 | '~/.duplicity_onedrive_oauthtoken.json') | ||
84 | 49 | |||
85 | 50 | API_URI = 'https://apis.live.net/v5.0/' | ||
86 | 51 | MAXIMUM_FRAGMENT_SIZE = 60 * 1024 * 1024 | ||
87 | 52 | BITS_1_5_UPLOAD_PROTOCOL = '{7df0354d-249b-430f-820d-3d2a9bef4931}' | ||
88 | 53 | CLIENT_ID = '000000004C12E85D' | ||
89 | 54 | CLIENT_SECRET = 'k1oR0CbtbvTG9nK1PEDeVW2dzvAaiN4d' | ||
90 | 55 | OAUTH_TOKEN_URI = 'https://login.live.com/oauth20_token.srf' | ||
91 | 56 | OAUTH_AUTHORIZE_URI = 'https://login.live.com/oauth20_authorize.srf' | ||
92 | 57 | OAUTH_REDIRECT_URI = 'https://login.live.com/oauth20_desktop.srf' | ||
93 | 58 | # wl.skydrive is for reading files, | ||
94 | 59 | # wl.skydrive_update is for creating/writing files, | ||
95 | 60 | # wl.offline_access is necessary for duplicity to access onedrive without | ||
96 | 61 | # the user being logged in right now. | ||
97 | 62 | OAUTH_SCOPE = ['wl.skydrive', 'wl.skydrive_update', 'wl.offline_access'] | ||
98 | 63 | |||
99 | 64 | def __init__(self, parsed_url): | ||
100 | 65 | duplicity.backend.Backend.__init__(self, parsed_url) | ||
101 | 66 | self.names_to_ids = None | ||
102 | 67 | self.user_id = None | ||
103 | 68 | self.directory = parsed_url.path.lstrip('/') | ||
104 | 69 | if self.directory == "": | ||
105 | 70 | raise BackendException(( | ||
106 | 71 | 'You did not specify a path. ' | ||
107 | 72 | 'Please specify a path, e.g. onedrive://duplicity_backups')) | ||
108 | 73 | if globals.volsize > (10 * 1024 * 1024 * 1024): | ||
109 | 74 | raise BackendException(( | ||
110 | 75 | 'Your --volsize is bigger than 10 GiB, which is the maximum ' | ||
111 | 76 | 'file size on OneDrive.')) | ||
112 | 77 | self.initialize_oauth2_session() | ||
113 | 78 | self.resolve_directory() | ||
114 | 79 | |||
115 | 80 | def initialize_oauth2_session(self): | ||
116 | 81 | def token_updater(token): | ||
117 | 82 | try: | ||
118 | 83 | with open(self.OAUTH_TOKEN_PATH, 'w') as f: | ||
119 | 84 | json.dump(token, f) | ||
120 | 85 | except Exception as e: | ||
121 | 86 | log.Error(('Could not save the OAuth2 token to %s. ' | ||
122 | 87 | 'This means you may need to do the OAuth2 ' | ||
123 | 88 | 'authorization process again soon. ' | ||
124 | 89 | 'Original error: %s' % ( | ||
125 | 90 | self.OAUTH_TOKEN_PATH, e))) | ||
126 | 91 | |||
127 | 92 | token = None | ||
128 | 93 | try: | ||
129 | 94 | with open(self.OAUTH_TOKEN_PATH) as f: | ||
130 | 95 | token = json.load(f) | ||
131 | 96 | except IOError as e: | ||
132 | 97 | log.Error(('Could not load OAuth2 token. ' | ||
133 | 98 | 'Trying to create a new one. (original error: %s)' % e)) | ||
134 | 99 | |||
135 | 100 | self.http_client = OAuth2Session( | ||
136 | 101 | self.CLIENT_ID, | ||
137 | 102 | scope=self.OAUTH_SCOPE, | ||
138 | 103 | redirect_uri=self.OAUTH_REDIRECT_URI, | ||
139 | 104 | token=token, | ||
140 | 105 | auto_refresh_kwargs={ | ||
141 | 106 | 'client_id': self.CLIENT_ID, | ||
142 | 107 | 'client_secret': self.CLIENT_SECRET, | ||
143 | 108 | }, | ||
144 | 109 | auto_refresh_url=self.OAUTH_TOKEN_URI, | ||
145 | 110 | token_updater=token_updater) | ||
146 | 111 | |||
147 | 112 | # Send a request to make sure the token is valid (or could at least be | ||
148 | 113 | # refreshed successfully, which will happen under the covers). In case | ||
149 | 114 | # this request fails, the provided token was too old (i.e. expired), | ||
150 | 115 | # and we need to get a new token. | ||
151 | 116 | user_info_response = self.http_client.get(self.API_URI + 'me') | ||
152 | 117 | if user_info_response.status_code != requests.codes.ok: | ||
153 | 118 | token = None | ||
154 | 119 | |||
155 | 120 | if token is None: | ||
156 | 121 | if not sys.stdout.isatty() or not sys.stdin.isatty(): | ||
157 | 122 | log.FatalError(('The OAuth2 token could not be loaded from %s ' | ||
158 | 123 | 'and you are not running duplicity ' | ||
159 | 124 | 'interactively, so duplicity cannot possibly ' | ||
160 | 125 | 'access OneDrive.' % self.OAUTH_TOKEN_PATH)) | ||
161 | 126 | authorization_url, state = self.http_client.authorization_url( | ||
162 | 127 | self.OAUTH_AUTHORIZE_URI, display='touch') | ||
163 | 128 | |||
164 | 129 | print '' | ||
165 | 130 | print ('In order to authorize duplicity to access your OneDrive, ' | ||
166 | 131 | 'please open %s in a browser and copy the URL of the blank ' | ||
167 | 132 | 'page the dialog leads to.' % authorization_url) | ||
168 | 133 | print '' | ||
169 | 134 | |||
170 | 135 | redirected_to = raw_input('URL of the blank page: ') | ||
171 | 136 | |||
172 | 137 | token = self.http_client.fetch_token(self.OAUTH_TOKEN_URI, | ||
173 | 138 | client_secret=self.CLIENT_SECRET, | ||
174 | 139 | authorization_response=redirected_to) | ||
175 | 140 | |||
176 | 141 | user_info_response = self.http_client.get(self.API_URI + 'me') | ||
177 | 142 | user_info_response.raise_for_status() | ||
178 | 143 | |||
179 | 144 | try: | ||
180 | 145 | with open(self.OAUTH_TOKEN_PATH, 'w') as f: | ||
181 | 146 | json.dump(token, f) | ||
182 | 147 | except Exception as e: | ||
183 | 148 | log.Error(('Could not save the OAuth2 token to %s. ' | ||
184 | 149 | 'This means you need to do the OAuth2 authorization ' | ||
185 | 150 | 'process on every start of duplicity. ' | ||
186 | 151 | 'Original error: %s' % ( | ||
187 | 152 | self.OAUTH_TOKEN_PATH, e))) | ||
188 | 153 | |||
189 | 154 | if not 'id' in user_info_response.json(): | ||
190 | 155 | log.Error('user info response lacks the "id" field.') | ||
191 | 156 | |||
192 | 157 | self.user_id = user_info_response.json()['id'] | ||
193 | 158 | |||
194 | 159 | def resolve_directory(self): | ||
195 | 160 | """Ensures self.directory_id contains the folder id for the path. | ||
196 | 161 | |||
197 | 162 | There is no API call to resolve a logical path (e.g. | ||
198 | 163 | /backups/duplicity/notebook/), so we recursively list directories | ||
199 | 164 | until we get the object id of the configured directory, creating | ||
200 | 165 | directories as necessary. | ||
201 | 166 | """ | ||
202 | 167 | object_id = 'me/skydrive' | ||
203 | 168 | for component in self.directory.split('/'): | ||
204 | 169 | tried_mkdir = False | ||
205 | 170 | while True: | ||
206 | 171 | files = self.get_files(object_id) | ||
207 | 172 | names_to_ids = {x['name']: x['id'] for x in files} | ||
208 | 173 | if component not in names_to_ids: | ||
209 | 174 | if not tried_mkdir: | ||
210 | 175 | self.mkdir(object_id, component) | ||
211 | 176 | tried_mkdir = True | ||
212 | 177 | continue | ||
213 | 178 | raise BackendException(( | ||
214 | 179 | 'Could not resolve/create directory "%s" on ' | ||
215 | 180 | 'OneDrive: %s not in %s (files of folder %s)' % ( | ||
216 | 181 | self.directory, component, | ||
217 | 182 | names_to_ids.keys(), object_id))) | ||
218 | 183 | break | ||
219 | 184 | object_id = names_to_ids[component] | ||
220 | 185 | self.directory_id = object_id | ||
221 | 186 | log.Debug('OneDrive id for the configured directory "%s" is "%s"' % ( | ||
222 | 187 | self.directory, self.directory_id)) | ||
223 | 188 | |||
224 | 189 | def mkdir(self, object_id, folder_name): | ||
225 | 190 | data = {'name': folder_name, 'description': 'Created by duplicity'} | ||
226 | 191 | headers = {'Content-Type': 'application/json'} | ||
227 | 192 | response = self.http_client.post( | ||
228 | 193 | self.API_URI + object_id, | ||
229 | 194 | data=json.dumps(data), | ||
230 | 195 | headers=headers) | ||
231 | 196 | response.raise_for_status() | ||
232 | 197 | |||
233 | 198 | def get_files(self, path): | ||
234 | 199 | response = self.http_client.get(self.API_URI + path + '/files') | ||
235 | 200 | response.raise_for_status() | ||
236 | 201 | if 'data' not in response.json(): | ||
237 | 202 | raise BackendException(( | ||
238 | 203 | 'Malformed JSON: expected "data" member in %s' % ( | ||
239 | 204 | response.json()))) | ||
240 | 205 | return response.json()['data'] | ||
241 | 206 | |||
242 | 207 | def _list(self): | ||
243 | 208 | files = self.get_files(self.directory_id) | ||
244 | 209 | self.names_to_ids = {x['name']: x['id'] for x in files} | ||
245 | 210 | return [x['name'] for x in files] | ||
246 | 211 | |||
247 | 212 | def get_file_id(self, remote_filename): | ||
248 | 213 | """Returns the file id from cache, updating the cache if necessary.""" | ||
249 | 214 | if (self.names_to_ids is None or | ||
250 | 215 | remote_filename not in self.names_to_ids): | ||
251 | 216 | self._list() | ||
252 | 217 | return self.names_to_ids.get(remote_filename) | ||
253 | 218 | |||
254 | 219 | def _get(self, remote_filename, local_path): | ||
255 | 220 | with local_path.open('wb') as f: | ||
256 | 221 | file_id = self.get_file_id(remote_filename) | ||
257 | 222 | if file_id is None: | ||
258 | 223 | raise BackendException(( | ||
259 | 224 | 'File "%s" cannot be downloaded: it does not exist' % ( | ||
260 | 225 | remote_filename))) | ||
261 | 226 | response = self.http_client.get( | ||
262 | 227 | self.API_URI + file_id + '/content', stream=True) | ||
263 | 228 | response.raise_for_status() | ||
264 | 229 | for chunk in response.iter_content(chunk_size=4096): | ||
265 | 230 | if chunk: | ||
266 | 231 | f.write(chunk) | ||
267 | 232 | f.flush() | ||
268 | 233 | |||
269 | 234 | def _put(self, source_path, remote_filename): | ||
270 | 235 | # Check if the user has enough space available on OneDrive before even | ||
271 | 236 | # attempting to upload the file. | ||
272 | 237 | source_size = os.path.getsize(source_path.name) | ||
273 | 238 | start = time.time() | ||
274 | 239 | response = self.http_client.get(self.API_URI + 'me/skydrive/quota') | ||
275 | 240 | response.raise_for_status() | ||
276 | 241 | if ('available' in response.json() and | ||
277 | 242 | source_size > response.json()['available']): | ||
278 | 243 | raise BackendException(( | ||
279 | 244 | 'Out of space: trying to store "%s" (%d bytes), but only ' | ||
280 | 245 | '%d bytes available on OneDrive.' % ( | ||
281 | 246 | source_path.name, source_size, | ||
282 | 247 | response.json()['available']))) | ||
283 | 248 | log.Debug("Checked quota in %fs" % (time.time() - start)) | ||
284 | 249 | |||
285 | 250 | with source_path.open() as source_file: | ||
286 | 251 | start = time.time() | ||
287 | 252 | # Create a BITS session, so that we can upload large files. | ||
288 | 253 | short_directory_id = self.directory_id.split('.')[-1] | ||
289 | 254 | url = 'https://cid-%s.users.storage.live.com/items/%s/%s' % ( | ||
290 | 255 | self.user_id, short_directory_id, remote_filename) | ||
291 | 256 | headers = { | ||
292 | 257 | 'X-Http-Method-Override': 'BITS_POST', | ||
293 | 258 | 'BITS-Packet-Type': 'Create-Session', | ||
294 | 259 | 'BITS-Supported-Protocols': self.BITS_1_5_UPLOAD_PROTOCOL, | ||
295 | 260 | } | ||
296 | 261 | |||
297 | 262 | response = self.http_client.post( | ||
298 | 263 | url, | ||
299 | 264 | headers=headers) | ||
300 | 265 | response.raise_for_status() | ||
301 | 266 | if (not 'bits-packet-type' in response.headers or | ||
302 | 267 | response.headers['bits-packet-type'].lower() != 'ack'): | ||
303 | 268 | raise BackendException(( | ||
304 | 269 | 'File "%s" cannot be uploaded: ' | ||
305 | 270 | 'Could not create BITS session: ' | ||
306 | 271 | 'Server response did not include BITS-Packet-Type: ACK' % ( | ||
307 | 272 | remote_filename))) | ||
308 | 273 | bits_session_id = response.headers['bits-session-id'] | ||
309 | 274 | log.Debug('BITS session id is "%s"' % bits_session_id) | ||
310 | 275 | |||
311 | 276 | # Send fragments (with a maximum size of 60 MB each). | ||
312 | 277 | offset = 0 | ||
313 | 278 | while True: | ||
314 | 279 | chunk = source_file.read(self.MAXIMUM_FRAGMENT_SIZE) | ||
315 | 280 | if len(chunk) == 0: | ||
316 | 281 | break | ||
317 | 282 | headers = { | ||
318 | 283 | 'X-Http-Method-Override': 'BITS_POST', | ||
319 | 284 | 'BITS-Packet-Type': 'Fragment', | ||
320 | 285 | 'BITS-Session-Id': bits_session_id, | ||
321 | 286 | 'Content-Range': 'bytes %d-%d/%d' % (offset, offset + len(chunk) - 1, source_size), | ||
322 | 287 | } | ||
323 | 288 | response = self.http_client.post( | ||
324 | 289 | url, | ||
325 | 290 | headers=headers, | ||
326 | 291 | data=chunk) | ||
327 | 292 | response.raise_for_status() | ||
328 | 293 | offset += len(chunk) | ||
329 | 294 | |||
330 | 295 | # Close the BITS session to commit the file. | ||
331 | 296 | headers = { | ||
332 | 297 | 'X-Http-Method-Override': 'BITS_POST', | ||
333 | 298 | 'BITS-Packet-Type': 'Close-Session', | ||
334 | 299 | 'BITS-Session-Id': bits_session_id, | ||
335 | 300 | } | ||
336 | 301 | response = self.http_client.post(url, headers=headers) | ||
337 | 302 | response.raise_for_status() | ||
338 | 303 | |||
339 | 304 | log.Debug("PUT file in %fs" % (time.time() - start)) | ||
340 | 305 | |||
341 | 306 | def _delete(self, remote_filename): | ||
342 | 307 | file_id = self.get_file_id(remote_filename) | ||
343 | 308 | if file_id is None: | ||
344 | 309 | raise BackendException(( | ||
345 | 310 | 'File "%s" cannot be deleted: it does not exist' % ( | ||
346 | 311 | remote_filename))) | ||
347 | 312 | response = self.http_client.delete(self.API_URI + file_id) | ||
348 | 313 | response.raise_for_status() | ||
349 | 314 | |||
350 | 315 | def _query(self, remote_filename): | ||
351 | 316 | file_id = self.get_file_id(remote_filename) | ||
352 | 317 | if file_id is None: | ||
353 | 318 | return {'size': -1} | ||
354 | 319 | response = self.http_client.get(self.API_URI + file_id) | ||
355 | 320 | response.raise_for_status() | ||
356 | 321 | if 'size' not in response.json(): | ||
357 | 322 | raise BackendException(( | ||
358 | 323 | 'Malformed JSON: expected "size" member in %s' % ( | ||
359 | 324 | response.json()))) | ||
360 | 325 | return {'size': response.json()['size']} | ||
361 | 326 | |||
362 | 327 | def _retry_cleanup(self): | ||
363 | 328 | self.initialize_oauth2_session() | ||
364 | 329 | |||
365 | 330 | duplicity.backend.register_backend('onedrive', OneDriveBackend) | ||
366 | 0 | 331 | ||
367 | === modified file 'duplicity/commandline.py' | |||
368 | --- duplicity/commandline.py 2014-12-12 14:54:56 +0000 | |||
369 | +++ duplicity/commandline.py 2015-01-04 21:11:21 +0000 | |||
370 | @@ -859,6 +859,7 @@ | |||
371 | 859 | mega://%(user)s[:%(password)s]@%(other_host)s/%(some_dir)s | 859 | mega://%(user)s[:%(password)s]@%(other_host)s/%(some_dir)s |
372 | 860 | copy://%(user)s[:%(password)s]@%(other_host)s/%(some_dir)s | 860 | copy://%(user)s[:%(password)s]@%(other_host)s/%(some_dir)s |
373 | 861 | dpbx:///%(some_dir)s | 861 | dpbx:///%(some_dir)s |
374 | 862 | onedrive://%(some_dir)s | ||
375 | 862 | 863 | ||
376 | 863 | """ % dict | 864 | """ % dict |
377 | 864 | 865 |