Merge lp:~matthew-t-bentley/duplicity/b2 into lp:~duplicity-team/duplicity/0.7-series
- b2
- Merge into 0.7-series
Status: | Superseded |
---|---|
Proposed branch: | lp:~matthew-t-bentley/duplicity/b2 |
Merge into: | lp:~duplicity-team/duplicity/0.7-series |
Diff against target: |
369 lines (+343/-0) 3 files modified
duplicity/backends/b2backend.py (+340/-0) duplicity/commandline.py (+1/-0) duplicity/log.py (+2/-0) |
To merge this branch: | bzr merge lp:~matthew-t-bentley/duplicity/b2 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
edso | Needs Fixing | ||
Review via email:
|
This proposal has been superseded by a proposal from 2015-12-03.
Commit message
Description of the change
Adds a backed for BackBlaze's (currently beta) B2 backup service.
This adds backends/
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
edso (ed.so) wrote : | # |
- 1157. By Matthew Bentley
-
Documentation
- 1158. By Matthew Bentley
-
Remove unnecessary requirements section in manpage
- 1159. By Matthew Bentley
-
Undo changes to po files
- 1160. By Matthew Bentley
-
Merge from upstream
- 1161. By Matthew Bentley
-
Fix missing import and typos
- 1162. By Matthew Bentley
-
Merge and re-add missing error
- 1163. By Matthew Bentley
-
Allow multiple backups in the same bucket
- 1164. By Matthew Bentley
-
Add debugging for b2backend
- 1165. By Matthew Bentley
-
Set fake (otherwise unused) hostname for prettier password prompt
Thanks to Andreas Knab <email address hidden> for the patch - 1166. By Matthew Bentley
-
Make sure listed files are exactly in the requested path
Thanks to Andreas Knab <email address hidden> for the pull request
Unmerged revisions
Preview Diff
1 | === added file 'duplicity/backends/b2backend.py' | |||
2 | --- duplicity/backends/b2backend.py 1970-01-01 00:00:00 +0000 | |||
3 | +++ duplicity/backends/b2backend.py 2015-12-02 21:59:48 +0000 | |||
4 | @@ -0,0 +1,340 @@ | |||
5 | 1 | # | ||
6 | 2 | # Copyright (c) 2015 Matthew Bentley | ||
7 | 3 | # | ||
8 | 4 | # | ||
9 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy | ||
10 | 6 | # of this software and associated documentation files (the "Software"), to deal | ||
11 | 7 | # in the Software without restriction, including without limitation the rights | ||
12 | 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
13 | 9 | # copies of the Software, and to permit persons to whom the Software is | ||
14 | 10 | # furnished to do so, subject to the following conditions: | ||
15 | 11 | # | ||
16 | 12 | # | ||
17 | 13 | # The above copyright notice and this permission notice shall be included in | ||
18 | 14 | # all copies or substantial portions of the Software. | ||
19 | 15 | # | ||
20 | 16 | # | ||
21 | 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
22 | 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
23 | 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
24 | 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
25 | 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
26 | 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | ||
27 | 23 | # THE SOFTWARE. | ||
28 | 24 | |||
29 | 25 | import os | ||
30 | 26 | import hashlib | ||
31 | 27 | |||
32 | 28 | import duplicity.backend | ||
33 | 29 | from duplicity.errors import BackendException, FatalBackendException | ||
34 | 30 | |||
35 | 31 | import json | ||
36 | 32 | import urllib2 | ||
37 | 33 | import base64 | ||
38 | 34 | |||
39 | 35 | |||
40 | 36 | class B2Backend(duplicity.backend.Backend): | ||
41 | 37 | """ | ||
42 | 38 | Backend for BackBlaze's B2 storage service | ||
43 | 39 | """ | ||
44 | 40 | |||
45 | 41 | def __init__(self, parsed_url): | ||
46 | 42 | """ | ||
47 | 43 | Authorize to B2 api and set up needed variables | ||
48 | 44 | """ | ||
49 | 45 | duplicity.backend.Backend.__init__(self, parsed_url) | ||
50 | 46 | |||
51 | 47 | self.account_id = parsed_url.username | ||
52 | 48 | account_key = self.get_password() | ||
53 | 49 | |||
54 | 50 | self.url_parts = [ | ||
55 | 51 | x for x in parsed_url.path.replace("@", "/").split('/') if x != '' | ||
56 | 52 | ] | ||
57 | 53 | if self.url_parts: | ||
58 | 54 | self.username = self.url_parts.pop(0) | ||
59 | 55 | self.bucket_name = self.url_parts.pop(0) | ||
60 | 56 | else: | ||
61 | 57 | raise BackendException("B2 requires a bucket name") | ||
62 | 58 | self.path = "/".join(self.url_parts) | ||
63 | 59 | |||
64 | 60 | id_and_key = self.account_id + ":" + account_key | ||
65 | 61 | basic_auth_string = 'Basic ' + base64.b64encode(id_and_key) | ||
66 | 62 | headers = {'Authorization': basic_auth_string} | ||
67 | 63 | |||
68 | 64 | request = urllib2.Request( | ||
69 | 65 | 'https://api.backblaze.com/b2api/v1/b2_authorize_account', | ||
70 | 66 | headers=headers | ||
71 | 67 | ) | ||
72 | 68 | |||
73 | 69 | response = urllib2.urlopen(request) | ||
74 | 70 | response_data = json.loads(response.read()) | ||
75 | 71 | response.close() | ||
76 | 72 | |||
77 | 73 | self.auth_token = response_data['authorizationToken'] | ||
78 | 74 | self.api_url = response_data['apiUrl'] | ||
79 | 75 | self.download_url = response_data['downloadUrl'] | ||
80 | 76 | |||
81 | 77 | try: | ||
82 | 78 | self.find_or_create_bucket(self.bucket_name) | ||
83 | 79 | except urllib2.HTTPError: | ||
84 | 80 | raise FatalBackendException("Bucket cannot be created") | ||
85 | 81 | |||
86 | 82 | def _get(self, remote_filename, local_path): | ||
87 | 83 | """ | ||
88 | 84 | Download remote_filename to local_path | ||
89 | 85 | """ | ||
90 | 86 | remote_filename = self.full_filename(remote_filename) | ||
91 | 87 | url = self.download_url + \ | ||
92 | 88 | '/file/' + self.bucket_name + '/' + \ | ||
93 | 89 | remote_filename | ||
94 | 90 | resp = self.get_or_post(url, None) | ||
95 | 91 | |||
96 | 92 | to_file = open(local_path.name, 'wb') | ||
97 | 93 | to_file.write(resp) | ||
98 | 94 | to_file.close() | ||
99 | 95 | |||
100 | 96 | def _put(self, source_path, remote_filename): | ||
101 | 97 | """ | ||
102 | 98 | Copy source_path to remote_filename | ||
103 | 99 | """ | ||
104 | 100 | self._delete(remote_filename) | ||
105 | 101 | digest = self.hex_sha1_of_file(source_path) | ||
106 | 102 | content_type = 'application/pgp-encrypted' | ||
107 | 103 | remote_filename = self.full_filename(remote_filename) | ||
108 | 104 | |||
109 | 105 | info = self.get_upload_info(self.bucket_id) | ||
110 | 106 | url = info['uploadUrl'] | ||
111 | 107 | |||
112 | 108 | headers = { | ||
113 | 109 | 'Authorization': info['authorizationToken'], | ||
114 | 110 | 'X-Bz-File-Name': remote_filename, | ||
115 | 111 | 'Content-Type': content_type, | ||
116 | 112 | 'X-Bz-Content-Sha1': digest, | ||
117 | 113 | 'Content-Length': str(os.path.getsize(source_path.name)), | ||
118 | 114 | } | ||
119 | 115 | data_file = source_path.open() | ||
120 | 116 | self.get_or_post(url, None, headers, data_file=data_file) | ||
121 | 117 | |||
122 | 118 | def _list(self): | ||
123 | 119 | """ | ||
124 | 120 | List files on remote server | ||
125 | 121 | """ | ||
126 | 122 | endpoint = 'b2_list_file_names' | ||
127 | 123 | url = self.formatted_url(endpoint) | ||
128 | 124 | params = { | ||
129 | 125 | 'bucketId': self.bucket_id, | ||
130 | 126 | 'maxFileCount': 1000, | ||
131 | 127 | } | ||
132 | 128 | try: | ||
133 | 129 | resp = self.get_or_post(url, params) | ||
134 | 130 | except urllib2.HTTPError: | ||
135 | 131 | return [] | ||
136 | 132 | |||
137 | 133 | files = [x['fileName'].split('/')[-1] for x in resp['files']] | ||
138 | 134 | |||
139 | 135 | next_file = resp['nextFileName'] | ||
140 | 136 | while next_file: | ||
141 | 137 | params['startFileName'] = next_file | ||
142 | 138 | try: | ||
143 | 139 | resp = self.get_or_post(url, params) | ||
144 | 140 | except urllib2.HTTPError: | ||
145 | 141 | return files | ||
146 | 142 | |||
147 | 143 | files += [x['fileName'].split('/')[-1] for x in resp['files']] | ||
148 | 144 | next_file = resp['nextFileName'] | ||
149 | 145 | |||
150 | 146 | return files | ||
151 | 147 | |||
152 | 148 | def _delete(self, filename): | ||
153 | 149 | """ | ||
154 | 150 | Delete filename from remote server | ||
155 | 151 | """ | ||
156 | 152 | endpoint = 'b2_delete_file_version' | ||
157 | 153 | url = self.formatted_url(endpoint) | ||
158 | 154 | fileid = self.get_file_id(filename) | ||
159 | 155 | if fileid is None: | ||
160 | 156 | return | ||
161 | 157 | filename = self.full_filename(filename) | ||
162 | 158 | params = {'fileName': filename, 'fileId': fileid} | ||
163 | 159 | try: | ||
164 | 160 | self.get_or_post(url, params) | ||
165 | 161 | except urllib2.HTTPError as e: | ||
166 | 162 | if e.code == 400: | ||
167 | 163 | return | ||
168 | 164 | else: | ||
169 | 165 | raise e | ||
170 | 166 | |||
171 | 167 | def _query(self, filename): | ||
172 | 168 | """ | ||
173 | 169 | Get size info of filename | ||
174 | 170 | """ | ||
175 | 171 | info = self.get_file_info(filename) | ||
176 | 172 | if not info: | ||
177 | 173 | return {'size': -1} | ||
178 | 174 | |||
179 | 175 | return {'size': info['size']} | ||
180 | 176 | |||
181 | 177 | def _error_code(self, operation, e): | ||
182 | 178 | if isinstance(e, urllib2.HTTPError): | ||
183 | 179 | if e.code == 400: | ||
184 | 180 | return log.ErrorCode.bad_request | ||
185 | 181 | if e.code == 500: | ||
186 | 182 | return log.ErrorCode.backed_error | ||
187 | 183 | if e.code == 403: | ||
188 | 184 | return log.ErrorCode.backed_permission_denied | ||
189 | 185 | |||
190 | 186 | def find_or_create_bucket(self, bucket_name): | ||
191 | 187 | """ | ||
192 | 188 | Find a bucket with name bucket_name and save its id. | ||
193 | 189 | If it doesn't exist, create it | ||
194 | 190 | """ | ||
195 | 191 | endpoint = 'b2_list_buckets' | ||
196 | 192 | url = self.formatted_url(endpoint) | ||
197 | 193 | |||
198 | 194 | params = {'accountId': self.account_id} | ||
199 | 195 | resp = self.get_or_post(url, params) | ||
200 | 196 | |||
201 | 197 | bucket_names = [x['bucketName'] for x in resp['buckets']] | ||
202 | 198 | |||
203 | 199 | if bucket_name not in bucket_names: | ||
204 | 200 | self.create_bucket(bucket_name) | ||
205 | 201 | else: | ||
206 | 202 | self.bucket_id = { | ||
207 | 203 | x[ | ||
208 | 204 | 'bucketName' | ||
209 | 205 | ]: x['bucketId'] for x in resp['buckets'] | ||
210 | 206 | }[bucket_name] | ||
211 | 207 | |||
212 | 208 | def create_bucket(self, bucket_name): | ||
213 | 209 | """ | ||
214 | 210 | Create a bucket with name bucket_name and save its id | ||
215 | 211 | """ | ||
216 | 212 | endpoint = 'b2_create_bucket' | ||
217 | 213 | url = self.formatted_url(endpoint) | ||
218 | 214 | params = { | ||
219 | 215 | 'accountId': self.account_id, | ||
220 | 216 | 'bucketName': bucket_name, | ||
221 | 217 | 'bucketType': 'allPrivate' | ||
222 | 218 | } | ||
223 | 219 | resp = self.get_or_post(url, params) | ||
224 | 220 | |||
225 | 221 | self.bucket_id = resp['bucketId'] | ||
226 | 222 | |||
227 | 223 | def formatted_url(self, endpoint): | ||
228 | 224 | """ | ||
229 | 225 | Return the full api endpoint from just the last part | ||
230 | 226 | """ | ||
231 | 227 | return '%s/b2api/v1/%s' % (self.api_url, endpoint) | ||
232 | 228 | |||
233 | 229 | def get_upload_info(self, bucket_id): | ||
234 | 230 | """ | ||
235 | 231 | Get an upload url for a bucket | ||
236 | 232 | """ | ||
237 | 233 | endpoint = 'b2_get_upload_url' | ||
238 | 234 | url = self.formatted_url(endpoint) | ||
239 | 235 | return self.get_or_post(url, {'bucketId': bucket_id}) | ||
240 | 236 | |||
241 | 237 | def get_or_post(self, url, data, headers=None, data_file=None): | ||
242 | 238 | """ | ||
243 | 239 | Sends the request, either get or post. | ||
244 | 240 | If data and data_file are None, send a get request. | ||
245 | 241 | data_file takes precedence over data. | ||
246 | 242 | If headers are not supplied, just send with an auth key | ||
247 | 243 | """ | ||
248 | 244 | if headers is None: | ||
249 | 245 | headers = {'Authorization': self.auth_token} | ||
250 | 246 | if data_file is not None: | ||
251 | 247 | data = data_file | ||
252 | 248 | else: | ||
253 | 249 | data = json.dumps(data) if data else None | ||
254 | 250 | |||
255 | 251 | encoded_headers = dict( | ||
256 | 252 | (k, urllib2.quote(v.encode('utf-8'))) | ||
257 | 253 | for (k, v) in headers.iteritems() | ||
258 | 254 | ) | ||
259 | 255 | |||
260 | 256 | with OpenUrl(url, data, encoded_headers) as resp: | ||
261 | 257 | out = resp.read() | ||
262 | 258 | try: | ||
263 | 259 | return json.loads(out) | ||
264 | 260 | except ValueError: | ||
265 | 261 | return out | ||
266 | 262 | |||
267 | 263 | def get_file_info(self, filename): | ||
268 | 264 | """ | ||
269 | 265 | Get a file info from filename | ||
270 | 266 | """ | ||
271 | 267 | endpoint = 'b2_list_file_names' | ||
272 | 268 | url = self.formatted_url(endpoint) | ||
273 | 269 | filename = self.full_filename(filename) | ||
274 | 270 | params = { | ||
275 | 271 | 'bucketId': self.bucket_id, | ||
276 | 272 | 'maxFileCount': 1, | ||
277 | 273 | 'startFileName': filename, | ||
278 | 274 | } | ||
279 | 275 | resp = self.get_or_post(url, params) | ||
280 | 276 | |||
281 | 277 | try: | ||
282 | 278 | return resp['files'][0] | ||
283 | 279 | except IndexError: | ||
284 | 280 | return None | ||
285 | 281 | except TypeError: | ||
286 | 282 | return None | ||
287 | 283 | |||
288 | 284 | def get_file_id(self, filename): | ||
289 | 285 | """ | ||
290 | 286 | Get a file id form filename | ||
291 | 287 | """ | ||
292 | 288 | try: | ||
293 | 289 | return self.get_file_info(filename)['fileId'] | ||
294 | 290 | except IndexError: | ||
295 | 291 | return None | ||
296 | 292 | except TypeError: | ||
297 | 293 | return None | ||
298 | 294 | |||
299 | 295 | def full_filename(self, filename): | ||
300 | 296 | if self.path: | ||
301 | 297 | return self.path + '/' + filename | ||
302 | 298 | else: | ||
303 | 299 | return filename | ||
304 | 300 | |||
305 | 301 | @staticmethod | ||
306 | 302 | def hex_sha1_of_file(path): | ||
307 | 303 | """ | ||
308 | 304 | Calculate the sha1 of a file to upload | ||
309 | 305 | """ | ||
310 | 306 | f = path.open() | ||
311 | 307 | block_size = 1024 * 1024 | ||
312 | 308 | digest = hashlib.sha1() | ||
313 | 309 | while True: | ||
314 | 310 | data = f.read(block_size) | ||
315 | 311 | if len(data) == 0: | ||
316 | 312 | break | ||
317 | 313 | digest.update(data) | ||
318 | 314 | f.close() | ||
319 | 315 | return digest.hexdigest() | ||
320 | 316 | |||
321 | 317 | |||
322 | 318 | class OpenUrl(object): | ||
323 | 319 | """ | ||
324 | 320 | Context manager that handles an open urllib2.Request, and provides | ||
325 | 321 | the file-like object that is the response. | ||
326 | 322 | """ | ||
327 | 323 | |||
328 | 324 | def __init__(self, url, data, headers): | ||
329 | 325 | self.url = url | ||
330 | 326 | self.data = data | ||
331 | 327 | self.headers = headers | ||
332 | 328 | self.file = None | ||
333 | 329 | |||
334 | 330 | def __enter__(self): | ||
335 | 331 | request = urllib2.Request(self.url, self.data, self.headers) | ||
336 | 332 | self.file = urllib2.urlopen(request) | ||
337 | 333 | return self.file | ||
338 | 334 | |||
339 | 335 | def __exit__(self, exception_type, exception, traceback): | ||
340 | 336 | if self.file is not None: | ||
341 | 337 | self.file.close() | ||
342 | 338 | |||
343 | 339 | |||
344 | 340 | duplicity.backend.register_backend("b2", B2Backend) | ||
345 | 0 | 341 | ||
346 | === modified file 'duplicity/commandline.py' | |||
347 | --- duplicity/commandline.py 2015-10-10 00:02:35 +0000 | |||
348 | +++ duplicity/commandline.py 2015-12-02 21:59:48 +0000 | |||
349 | @@ -910,6 +910,7 @@ | |||
350 | 910 | dpbx:///%(some_dir)s | 910 | dpbx:///%(some_dir)s |
351 | 911 | onedrive://%(some_dir)s | 911 | onedrive://%(some_dir)s |
352 | 912 | azure://%(container_name)s | 912 | azure://%(container_name)s |
353 | 913 | b2://%(user)s@%(bucket_name)s/[%(some_dir)s/] | ||
354 | 913 | 914 | ||
355 | 914 | """ % dict | 915 | """ % dict |
356 | 915 | 916 | ||
357 | 916 | 917 | ||
358 | === modified file 'duplicity/log.py' | |||
359 | --- duplicity/log.py 2015-05-08 12:28:47 +0000 | |||
360 | +++ duplicity/log.py 2015-12-02 21:59:48 +0000 | |||
361 | @@ -307,6 +307,8 @@ | |||
362 | 307 | 307 | ||
363 | 308 | dpbx_nologin = 47 | 308 | dpbx_nologin = 47 |
364 | 309 | 309 | ||
365 | 310 | bad_request = 48 | ||
366 | 311 | |||
367 | 310 | # 50->69 reserved for backend errors | 312 | # 50->69 reserved for backend errors |
368 | 311 | backend_error = 50 | 313 | backend_error = 50 |
369 | 312 | backend_permission_denied = 51 | 314 | backend_permission_denied = 51 |
please add backend documentation to the manpage (sections Requirements, Url Formats etc.)
bin/duplicity.1
..ede/duply.net