Merge lp:~trb143/openlp/httpfixes into lp:openlp

Proposed by Tim Bentley
Status: Superseded
Proposed branch: lp:~trb143/openlp/httpfixes
Merge into: lp:openlp
Diff against target: 1181 lines (+541/-510)
8 files modified
openlp/core/common/httputils.py (+255/-0)
openlp/core/lib/webpagereader.py (+0/-182)
openlp/core/ui/firsttimeform.py (+10/-79)
openlp/plugins/bibles/lib/importers/http.py (+1/-1)
tests/functional/openlp_core_common/test_httputils.py (+274/-0)
tests/functional/openlp_core_lib/test_webpagereader.py (+0/-229)
tests/functional/openlp_core_ui/test_first_time.py (+1/-1)
tests/functional/openlp_core_ui/test_firsttimeform.py (+0/-18)
To merge this branch: bzr merge lp:~trb143/openlp/httpfixes
Reviewer Review Type Date Requested Status
OpenLP Core Pending
Review via email: mp+313677@code.launchpad.net

This proposal has been superseded by a proposal from 2016-12-21.

To post a comment you must log in.
lp:~trb143/openlp/httpfixes updated
2719. By Tim Bentley

missed one

2720. By Tim Bentley

Remove extra self

Unmerged revisions

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added file 'openlp/core/common/httputils.py'
2--- openlp/core/common/httputils.py 1970-01-01 00:00:00 +0000
3+++ openlp/core/common/httputils.py 2016-12-21 10:20:49 +0000
4@@ -0,0 +1,255 @@
5+# -*- coding: utf-8 -*-
6+# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
7+
8+###############################################################################
9+# OpenLP - Open Source Lyrics Projection #
10+# --------------------------------------------------------------------------- #
11+# Copyright (c) 2008-2016 OpenLP Developers #
12+# --------------------------------------------------------------------------- #
13+# This program is free software; you can redistribute it and/or modify it #
14+# under the terms of the GNU General Public License as published by the Free #
15+# Software Foundation; version 2 of the License. #
16+# #
17+# This program is distributed in the hope that it will be useful, but WITHOUT #
18+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
19+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
20+# more details. #
21+# #
22+# You should have received a copy of the GNU General Public License along #
23+# with this program; if not, write to the Free Software Foundation, Inc., 59 #
24+# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
25+###############################################################################
26+"""
27+The :mod:`openlp.core.utils` module provides the utility libraries for OpenLP.
28+"""
29+import hashlib
30+import logging
31+import os
32+import socket
33+import sys
34+import time
35+import urllib.error
36+import urllib.parse
37+import urllib.request
38+from http.client import HTTPException
39+from random import randint
40+
41+from openlp.core.common import Registry, trace_error_handler
42+
43+log = logging.getLogger(__name__ + '.__init__')
44+
45+USER_AGENTS = {
46+ 'win32': [
47+ 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
48+ 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
49+ 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36'
50+ ],
51+ 'darwin': [
52+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) '
53+ 'Chrome/26.0.1410.43 Safari/537.31',
54+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/536.11 (KHTML, like Gecko) '
55+ 'Chrome/20.0.1132.57 Safari/536.11',
56+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/536.11 (KHTML, like Gecko) '
57+ 'Chrome/20.0.1132.47 Safari/536.11',
58+ ],
59+ 'linux2': [
60+ 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 '
61+ 'Chrome/25.0.1364.160 Safari/537.22',
62+ 'Mozilla/5.0 (X11; CrOS armv7l 2913.260.0) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.99 '
63+ 'Safari/537.11',
64+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.27 (KHTML, like Gecko) Chrome/26.0.1389.0 Safari/537.27'
65+ ],
66+ 'default': [
67+ 'Mozilla/5.0 (X11; NetBSD amd64; rv:18.0) Gecko/20130120 Firefox/18.0'
68+ ]
69+}
70+CONNECTION_TIMEOUT = 30
71+CONNECTION_RETRIES = 2
72+
73+
74+class HTTPRedirectHandlerFixed(urllib.request.HTTPRedirectHandler):
75+ """
76+ Special HTTPRedirectHandler used to work around http://bugs.python.org/issue22248
77+ (Redirecting to urls with special chars)
78+ """
79+ def redirect_request(self, req, fp, code, msg, headers, new_url):
80+ #
81+ """
82+ Test if the new_url can be decoded to ascii
83+
84+ :param req:
85+ :param fp:
86+ :param code:
87+ :param msg:
88+ :param headers:
89+ :param new_url:
90+ :return:
91+ """
92+ try:
93+ new_url.encode('latin1').decode('ascii')
94+ fixed_url = new_url
95+ except Exception:
96+ # The url could not be decoded to ascii, so we do some url encoding
97+ fixed_url = urllib.parse.quote(new_url.encode('latin1').decode('utf-8', 'replace'), safe='/:')
98+ return super(HTTPRedirectHandlerFixed, self).redirect_request(req, fp, code, msg, headers, fixed_url)
99+
100+
101+def get_user_agent():
102+ """
103+ Return a user agent customised for the platform the user is on.
104+ """
105+ browser_list = USER_AGENTS.get(sys.platform, None)
106+ if not browser_list:
107+ browser_list = USER_AGENTS['default']
108+ random_index = randint(0, len(browser_list) - 1)
109+ return browser_list[random_index]
110+
111+
112+def get_web_page(url, header=None, update_openlp=False):
113+ """
114+ Attempts to download the webpage at url and returns that page or None.
115+
116+ :param url: The URL to be downloaded.
117+ :param header: An optional HTTP header to pass in the request to the web server.
118+ :param update_openlp: Tells OpenLP to update itself if the page is successfully downloaded.
119+ Defaults to False.
120+ """
121+ # TODO: Add proxy usage. Get proxy info from OpenLP settings, add to a
122+ # proxy_handler, build into an opener and install the opener into urllib2.
123+ # http://docs.python.org/library/urllib2.html
124+ if not url:
125+ return None
126+ # This is needed to work around http://bugs.python.org/issue22248 and https://bugs.launchpad.net/openlp/+bug/1251437
127+ opener = urllib.request.build_opener(HTTPRedirectHandlerFixed())
128+ urllib.request.install_opener(opener)
129+ req = urllib.request.Request(url)
130+ if not header or header[0].lower() != 'user-agent':
131+ user_agent = get_user_agent()
132+ req.add_header('User-Agent', user_agent)
133+ if header:
134+ req.add_header(header[0], header[1])
135+ log.debug('Downloading URL = %s' % url)
136+ retries = 0
137+ while retries <= CONNECTION_RETRIES:
138+ retries += 1
139+ time.sleep(0.1)
140+ try:
141+ page = urllib.request.urlopen(req, timeout=CONNECTION_TIMEOUT)
142+ log.debug('Downloaded page {text}'.format(text=page.geturl()))
143+ break
144+ except urllib.error.URLError as err:
145+ log.exception('URLError on {text}'.format(text=url))
146+ log.exception('URLError: {text}'.format(text=err.reason))
147+ page = None
148+ if retries > CONNECTION_RETRIES:
149+ raise
150+ except socket.timeout:
151+ log.exception('Socket timeout: {text}'.format(text=url))
152+ page = None
153+ if retries > CONNECTION_RETRIES:
154+ raise
155+ except socket.gaierror:
156+ log.exception('Socket gaierror: {text}'.format(text=url))
157+ page = None
158+ if retries > CONNECTION_RETRIES:
159+ raise
160+ except ConnectionRefusedError:
161+ log.exception('ConnectionRefused: {text}'.format(text=url))
162+ page = None
163+ if retries > CONNECTION_RETRIES:
164+ raise
165+ break
166+ except ConnectionError:
167+ log.exception('Connection error: {text}'.format(text=url))
168+ page = None
169+ if retries > CONNECTION_RETRIES:
170+ raise
171+ except HTTPException:
172+ log.exception('HTTPException error: {text}'.format(text=url))
173+ page = None
174+ if retries > CONNECTION_RETRIES:
175+ raise
176+ except:
177+ # Don't know what's happening, so reraise the original
178+ raise
179+ if update_openlp:
180+ Registry().get('application').process_events()
181+ if not page:
182+ log.exception('{text} could not be downloaded'.format(text=url))
183+ return None
184+ log.debug(page)
185+ return page
186+
187+
188+def get_url_file_size(url):
189+ """
190+ Get the size of a file.
191+
192+ :param url: The URL of the file we want to download.
193+ """
194+ retries = 0
195+ while True:
196+ try:
197+ site = urllib.request.urlopen(url, timeout=CONNECTION_TIMEOUT)
198+ meta = site.info()
199+ return int(meta.get("Content-Length"))
200+ except urllib.error.URLError:
201+ if retries > CONNECTION_RETRIES:
202+ raise
203+ else:
204+ retries += 1
205+ time.sleep(0.1)
206+ continue
207+
208+
209+def url_get_file(callback, url, f_path, sha256=None):
210+ """"
211+ Download a file given a URL. The file is retrieved in chunks, giving the ability to cancel the download at any
212+ point. Returns False on download error.
213+
214+ :param callback: the class which needs to be updated
215+ :param url: URL to download
216+ :param f_path: Destination file
217+ :param sha256: The check sum value to be checked against the download value
218+ """
219+ block_count = 0
220+ block_size = 4096
221+ retries = 0
222+ while True:
223+ try:
224+ filename = open(f_path, "wb")
225+ url_file = urllib.request.urlopen(url, timeout=CONNECTION_TIMEOUT)
226+ if sha256:
227+ hasher = hashlib.sha256()
228+ # Download until finished or canceled.
229+ while not callback.was_cancelled:
230+ data = url_file.read(block_size)
231+ if not data:
232+ break
233+ filename.write(data)
234+ if sha256:
235+ hasher.update(data)
236+ block_count += 1
237+ callback._download_progress(block_count, block_size)
238+ filename.close()
239+ if sha256 and hasher.hexdigest() != sha256:
240+ log.error('sha256 sums did not match for file: {file}'.format(file=f_path))
241+ os.remove(f_path)
242+ return False
243+ except (urllib.error.URLError, socket.timeout) as err:
244+ trace_error_handler(log)
245+ filename.close()
246+ os.remove(f_path)
247+ if retries > CONNECTION_RETRIES:
248+ return False
249+ else:
250+ retries += 1
251+ time.sleep(0.1)
252+ continue
253+ break
254+ # Delete file if cancelled, it may be a partial file.
255+ if callback.was_cancelled:
256+ os.remove(f_path)
257+ return True
258+
259+__all__ = ['get_web_page']
260
261=== removed file 'openlp/core/lib/webpagereader.py'
262--- openlp/core/lib/webpagereader.py 2016-07-01 21:17:20 +0000
263+++ openlp/core/lib/webpagereader.py 1970-01-01 00:00:00 +0000
264@@ -1,182 +0,0 @@
265-# -*- coding: utf-8 -*-
266-# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
267-
268-###############################################################################
269-# OpenLP - Open Source Lyrics Projection #
270-# --------------------------------------------------------------------------- #
271-# Copyright (c) 2008-2016 OpenLP Developers #
272-# --------------------------------------------------------------------------- #
273-# This program is free software; you can redistribute it and/or modify it #
274-# under the terms of the GNU General Public License as published by the Free #
275-# Software Foundation; version 2 of the License. #
276-# #
277-# This program is distributed in the hope that it will be useful, but WITHOUT #
278-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
279-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
280-# more details. #
281-# #
282-# You should have received a copy of the GNU General Public License along #
283-# with this program; if not, write to the Free Software Foundation, Inc., 59 #
284-# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
285-###############################################################################
286-"""
287-The :mod:`openlp.core.utils` module provides the utility libraries for OpenLP.
288-"""
289-import logging
290-import socket
291-import sys
292-import time
293-import urllib.error
294-import urllib.parse
295-import urllib.request
296-from http.client import HTTPException
297-from random import randint
298-
299-from openlp.core.common import Registry
300-
301-log = logging.getLogger(__name__ + '.__init__')
302-
303-USER_AGENTS = {
304- 'win32': [
305- 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
306- 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
307- 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36'
308- ],
309- 'darwin': [
310- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) '
311- 'Chrome/26.0.1410.43 Safari/537.31',
312- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/536.11 (KHTML, like Gecko) '
313- 'Chrome/20.0.1132.57 Safari/536.11',
314- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/536.11 (KHTML, like Gecko) '
315- 'Chrome/20.0.1132.47 Safari/536.11',
316- ],
317- 'linux2': [
318- 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 '
319- 'Chrome/25.0.1364.160 Safari/537.22',
320- 'Mozilla/5.0 (X11; CrOS armv7l 2913.260.0) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.99 '
321- 'Safari/537.11',
322- 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.27 (KHTML, like Gecko) Chrome/26.0.1389.0 Safari/537.27'
323- ],
324- 'default': [
325- 'Mozilla/5.0 (X11; NetBSD amd64; rv:18.0) Gecko/20130120 Firefox/18.0'
326- ]
327-}
328-CONNECTION_TIMEOUT = 30
329-CONNECTION_RETRIES = 2
330-
331-
332-class HTTPRedirectHandlerFixed(urllib.request.HTTPRedirectHandler):
333- """
334- Special HTTPRedirectHandler used to work around http://bugs.python.org/issue22248
335- (Redirecting to urls with special chars)
336- """
337- def redirect_request(self, req, fp, code, msg, headers, new_url):
338- #
339- """
340- Test if the new_url can be decoded to ascii
341-
342- :param req:
343- :param fp:
344- :param code:
345- :param msg:
346- :param headers:
347- :param new_url:
348- :return:
349- """
350- try:
351- new_url.encode('latin1').decode('ascii')
352- fixed_url = new_url
353- except Exception:
354- # The url could not be decoded to ascii, so we do some url encoding
355- fixed_url = urllib.parse.quote(new_url.encode('latin1').decode('utf-8', 'replace'), safe='/:')
356- return super(HTTPRedirectHandlerFixed, self).redirect_request(req, fp, code, msg, headers, fixed_url)
357-
358-
359-def _get_user_agent():
360- """
361- Return a user agent customised for the platform the user is on.
362- """
363- browser_list = USER_AGENTS.get(sys.platform, None)
364- if not browser_list:
365- browser_list = USER_AGENTS['default']
366- random_index = randint(0, len(browser_list) - 1)
367- return browser_list[random_index]
368-
369-
370-def get_web_page(url, header=None, update_openlp=False):
371- """
372- Attempts to download the webpage at url and returns that page or None.
373-
374- :param url: The URL to be downloaded.
375- :param header: An optional HTTP header to pass in the request to the web server.
376- :param update_openlp: Tells OpenLP to update itself if the page is successfully downloaded.
377- Defaults to False.
378- """
379- # TODO: Add proxy usage. Get proxy info from OpenLP settings, add to a
380- # proxy_handler, build into an opener and install the opener into urllib2.
381- # http://docs.python.org/library/urllib2.html
382- if not url:
383- return None
384- # This is needed to work around http://bugs.python.org/issue22248 and https://bugs.launchpad.net/openlp/+bug/1251437
385- opener = urllib.request.build_opener(HTTPRedirectHandlerFixed())
386- urllib.request.install_opener(opener)
387- req = urllib.request.Request(url)
388- if not header or header[0].lower() != 'user-agent':
389- user_agent = _get_user_agent()
390- req.add_header('User-Agent', user_agent)
391- if header:
392- req.add_header(header[0], header[1])
393- log.debug('Downloading URL = %s' % url)
394- retries = 0
395- while retries <= CONNECTION_RETRIES:
396- retries += 1
397- time.sleep(0.1)
398- try:
399- page = urllib.request.urlopen(req, timeout=CONNECTION_TIMEOUT)
400- log.debug('Downloaded page {text}'.format(text=page.geturl()))
401- break
402- except urllib.error.URLError as err:
403- log.exception('URLError on {text}'.format(text=url))
404- log.exception('URLError: {text}'.format(text=err.reason))
405- page = None
406- if retries > CONNECTION_RETRIES:
407- raise
408- except socket.timeout:
409- log.exception('Socket timeout: {text}'.format(text=url))
410- page = None
411- if retries > CONNECTION_RETRIES:
412- raise
413- except socket.gaierror:
414- log.exception('Socket gaierror: {text}'.format(text=url))
415- page = None
416- if retries > CONNECTION_RETRIES:
417- raise
418- except ConnectionRefusedError:
419- log.exception('ConnectionRefused: {text}'.format(text=url))
420- page = None
421- if retries > CONNECTION_RETRIES:
422- raise
423- break
424- except ConnectionError:
425- log.exception('Connection error: {text}'.format(text=url))
426- page = None
427- if retries > CONNECTION_RETRIES:
428- raise
429- except HTTPException:
430- log.exception('HTTPException error: {text}'.format(text=url))
431- page = None
432- if retries > CONNECTION_RETRIES:
433- raise
434- except:
435- # Don't know what's happening, so reraise the original
436- raise
437- if update_openlp:
438- Registry().get('application').process_events()
439- if not page:
440- log.exception('{text} could not be downloaded'.format(text=url))
441- return None
442- log.debug(page)
443- return page
444-
445-
446-__all__ = ['get_web_page']
447
448=== modified file 'openlp/core/ui/firsttimeform.py'
449--- openlp/core/ui/firsttimeform.py 2016-08-05 19:41:22 +0000
450+++ openlp/core/ui/firsttimeform.py 2016-12-21 10:20:49 +0000
451@@ -22,7 +22,6 @@
452 """
453 This module contains the first time wizard.
454 """
455-import hashlib
456 import logging
457 import os
458 import socket
459@@ -39,7 +38,7 @@
460 translate, clean_button_text, trace_error_handler
461 from openlp.core.lib import PluginStatus, build_icon
462 from openlp.core.lib.ui import critical_error_message_box
463-from openlp.core.lib.webpagereader import get_web_page, CONNECTION_RETRIES, CONNECTION_TIMEOUT
464+from openlp.core.common.httputils import get_web_page, get_url_file_size, url_get_file, CONNECTION_TIMEOUT
465 from .firsttimewizard import UiFirstTimeWizard, FirstTimePage
466
467 log = logging.getLogger(__name__)
468@@ -395,54 +394,6 @@
469 self.was_cancelled = True
470 self.close()
471
472- def url_get_file(self, url, f_path, sha256=None):
473- """"
474- Download a file given a URL. The file is retrieved in chunks, giving the ability to cancel the download at any
475- point. Returns False on download error.
476-
477- :param url: URL to download
478- :param f_path: Destination file
479- """
480- block_count = 0
481- block_size = 4096
482- retries = 0
483- while True:
484- try:
485- filename = open(f_path, "wb")
486- url_file = urllib.request.urlopen(url, timeout=CONNECTION_TIMEOUT)
487- if sha256:
488- hasher = hashlib.sha256()
489- # Download until finished or canceled.
490- while not self.was_cancelled:
491- data = url_file.read(block_size)
492- if not data:
493- break
494- filename.write(data)
495- if sha256:
496- hasher.update(data)
497- block_count += 1
498- self._download_progress(block_count, block_size)
499- filename.close()
500- if sha256 and hasher.hexdigest() != sha256:
501- log.error('sha256 sums did not match for file: {file}'.format(file=f_path))
502- os.remove(f_path)
503- return False
504- except (urllib.error.URLError, socket.timeout) as err:
505- trace_error_handler(log)
506- filename.close()
507- os.remove(f_path)
508- if retries > CONNECTION_RETRIES:
509- return False
510- else:
511- retries += 1
512- time.sleep(0.1)
513- continue
514- break
515- # Delete file if cancelled, it may be a partial file.
516- if self.was_cancelled:
517- os.remove(f_path)
518- return True
519-
520 def _build_theme_screenshots(self):
521 """
522 This method builds the theme screenshots' icons for all items in the ``self.themes_list_widget``.
523@@ -455,26 +406,6 @@
524 if item:
525 item.setIcon(build_icon(os.path.join(gettempdir(), 'openlp', screenshot)))
526
527- def _get_file_size(self, url):
528- """
529- Get the size of a file.
530-
531- :param url: The URL of the file we want to download.
532- """
533- retries = 0
534- while True:
535- try:
536- site = urllib.request.urlopen(url, timeout=CONNECTION_TIMEOUT)
537- meta = site.info()
538- return int(meta.get("Content-Length"))
539- except urllib.error.URLError:
540- if retries > CONNECTION_RETRIES:
541- raise
542- else:
543- retries += 1
544- time.sleep(0.1)
545- continue
546-
547 def _download_progress(self, count, block_size):
548 """
549 Calculate and display the download progress.
550@@ -510,7 +441,7 @@
551 item = self.songs_list_widget.item(i)
552 if item.checkState() == QtCore.Qt.Checked:
553 filename, sha256 = item.data(QtCore.Qt.UserRole)
554- size = self._get_file_size('{path}{name}'.format(path=self.songs_url, name=filename))
555+ size = get_url_file_size('{path}{name}'.format(path=self.songs_url, name=filename))
556 self.max_progress += size
557 # Loop through the Bibles list and increase for each selected item
558 iterator = QtWidgets.QTreeWidgetItemIterator(self.bibles_tree_widget)
559@@ -519,7 +450,7 @@
560 item = iterator.value()
561 if item.parent() and item.checkState(0) == QtCore.Qt.Checked:
562 filename, sha256 = item.data(0, QtCore.Qt.UserRole)
563- size = self._get_file_size('{path}{name}'.format(path=self.bibles_url, name=filename))
564+ size = get_url_file_size('{path}{name}'.format(path=self.bibles_url, name=filename))
565 self.max_progress += size
566 iterator += 1
567 # Loop through the themes list and increase for each selected item
568@@ -528,7 +459,7 @@
569 item = self.themes_list_widget.item(i)
570 if item.checkState() == QtCore.Qt.Checked:
571 filename, sha256 = item.data(QtCore.Qt.UserRole)
572- size = self._get_file_size('{path}{name}'.format(path=self.themes_url, name=filename))
573+ size = get_url_file_size('{path}{name}'.format(path=self.themes_url, name=filename))
574 self.max_progress += size
575 except urllib.error.URLError:
576 trace_error_handler(log)
577@@ -636,8 +567,8 @@
578 self._increment_progress_bar(self.downloading.format(name=filename), 0)
579 self.previous_size = 0
580 destination = os.path.join(songs_destination, str(filename))
581- if not self.url_get_file('{path}{name}'.format(path=self.songs_url, name=filename),
582- destination, sha256):
583+ if not url_get_file(self, '{path}{name}'.format(path=self.songs_url, name=filename),
584+ destination, sha256):
585 missed_files.append('Song: {name}'.format(name=filename))
586 # Download Bibles
587 bibles_iterator = QtWidgets.QTreeWidgetItemIterator(self.bibles_tree_widget)
588@@ -648,9 +579,9 @@
589 # TODO: Tested at home
590 self._increment_progress_bar(self.downloading.format(name=bible), 0)
591 self.previous_size = 0
592- if not self.url_get_file('{path}{name}'.format(path=self.bibles_url, name=bible),
593- os.path.join(bibles_destination, bible),
594- sha256):
595+ if not url_get_file(self, '{path}{name}'.format(path=self.bibles_url, name=bible),
596+ os.path.join(bibles_destination, bible),
597+ sha256):
598 missed_files.append('Bible: {name}'.format(name=bible))
599 bibles_iterator += 1
600 # Download themes
601@@ -661,7 +592,7 @@
602 # TODO: Tested at home
603 self._increment_progress_bar(self.downloading.format(name=theme), 0)
604 self.previous_size = 0
605- if not self.url_get_file('{path}{name}'.format(path=self.themes_url, name=theme),
606+ if not self.url_get_file(self, '{path}{name}'.format(path=self.themes_url, name=theme),
607 os.path.join(themes_destination, theme),
608 sha256):
609 missed_files.append('Theme: {name}'.format(name=theme))
610
611=== modified file 'openlp/plugins/bibles/lib/importers/http.py'
612--- openlp/plugins/bibles/lib/importers/http.py 2016-09-18 14:54:55 +0000
613+++ openlp/plugins/bibles/lib/importers/http.py 2016-12-21 10:20:49 +0000
614@@ -32,7 +32,7 @@
615
616 from openlp.core.common import Registry, RegistryProperties, translate
617 from openlp.core.lib.ui import critical_error_message_box
618-from openlp.core.lib.webpagereader import get_web_page
619+from openlp.core.common.httputils import get_web_page
620 from openlp.plugins.bibles.lib import SearchResults
621 from openlp.plugins.bibles.lib.bibleimport import BibleImport
622 from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB, Book
623
624=== added file 'tests/functional/openlp_core_common/test_httputils.py'
625--- tests/functional/openlp_core_common/test_httputils.py 1970-01-01 00:00:00 +0000
626+++ tests/functional/openlp_core_common/test_httputils.py 2016-12-21 10:20:49 +0000
627@@ -0,0 +1,274 @@
628+# -*- coding: utf-8 -*-
629+# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
630+
631+###############################################################################
632+# OpenLP - Open Source Lyrics Projection #
633+# --------------------------------------------------------------------------- #
634+# Copyright (c) 2008-2016 OpenLP Developers #
635+# --------------------------------------------------------------------------- #
636+# This program is free software; you can redistribute it and/or modify it #
637+# under the terms of the GNU General Public License as published by the Free #
638+# Software Foundation; version 2 of the License. #
639+# #
640+# This program is distributed in the hope that it will be useful, but WITHOUT #
641+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
642+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
643+# more details. #
644+# #
645+# You should have received a copy of the GNU General Public License along #
646+# with this program; if not, write to the Free Software Foundation, Inc., 59 #
647+# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
648+###############################################################################
649+"""
650+Functional tests to test the AppLocation class and related methods.
651+"""
652+import os
653+import tempfile
654+import socket
655+from unittest import TestCase
656+
657+from openlp.core.common.httputils import get_user_agent, get_web_page, get_url_file_size, url_get_file
658+
659+from tests.functional import MagicMock, patch
660+from tests.helpers.testmixin import TestMixin
661+
662+
663+class TestHttpUtils(TestCase, TestMixin):
664+
665+ """
666+ A test suite to test out various http helper functions.
667+ """
668+ def setUp(self):
669+ self.tempfile = os.path.join(tempfile.gettempdir(), 'testfile')
670+
671+ def tearDown(self):
672+ if os.path.isfile(self.tempfile):
673+ os.remove(self.tempfile)
674+
675+ def test_get_user_agent_linux(self):
676+ """
677+ Test that getting a user agent on Linux returns a user agent suitable for Linux
678+ """
679+ with patch('openlp.core.common.httputils.sys') as mocked_sys:
680+
681+ # GIVEN: The system is Linux
682+ mocked_sys.platform = 'linux2'
683+
684+ # WHEN: We call get_user_agent()
685+ user_agent = get_user_agent()
686+
687+ # THEN: The user agent is a Linux (or ChromeOS) user agent
688+ result = 'Linux' in user_agent or 'CrOS' in user_agent
689+ self.assertTrue(result, 'The user agent should be a valid Linux user agent')
690+
691+ def test_get_user_agent_windows(self):
692+ """
693+ Test that getting a user agent on Windows returns a user agent suitable for Windows
694+ """
695+ with patch('openlp.core.common.httputils.sys') as mocked_sys:
696+
697+ # GIVEN: The system is Linux
698+ mocked_sys.platform = 'win32'
699+
700+ # WHEN: We call get_user_agent()
701+ user_agent = get_user_agent()
702+
703+ # THEN: The user agent is a Linux (or ChromeOS) user agent
704+ self.assertIn('Windows', user_agent, 'The user agent should be a valid Windows user agent')
705+
706+ def test_get_user_agent_macos(self):
707+ """
708+ Test that getting a user agent on OS X returns a user agent suitable for OS X
709+ """
710+ with patch('openlp.core.common.httputils.sys') as mocked_sys:
711+
712+ # GIVEN: The system is Linux
713+ mocked_sys.platform = 'darwin'
714+
715+ # WHEN: We call get_user_agent()
716+ user_agent = get_user_agent()
717+
718+ # THEN: The user agent is a Linux (or ChromeOS) user agent
719+ self.assertIn('Mac OS X', user_agent, 'The user agent should be a valid OS X user agent')
720+
721+ def test_get_user_agent_default(self):
722+ """
723+ Test that getting a user agent on a non-Linux/Windows/OS X platform returns the default user agent
724+ """
725+ with patch('openlp.core.common.httputils.sys') as mocked_sys:
726+
727+ # GIVEN: The system is Linux
728+ mocked_sys.platform = 'freebsd'
729+
730+ # WHEN: We call get_user_agent()
731+ user_agent = get_user_agent()
732+
733+ # THEN: The user agent is a Linux (or ChromeOS) user agent
734+ self.assertIn('NetBSD', user_agent, 'The user agent should be the default user agent')
735+
736+ def test_get_web_page_no_url(self):
737+ """
738+ Test that sending a URL of None to the get_web_page method returns None
739+ """
740+ # GIVEN: A None url
741+ test_url = None
742+
743+ # WHEN: We try to get the test URL
744+ result = get_web_page(test_url)
745+
746+ # THEN: None should be returned
747+ self.assertIsNone(result, 'The return value of get_web_page should be None')
748+
749+ def test_get_web_page(self):
750+ """
751+ Test that the get_web_page method works correctly
752+ """
753+ with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \
754+ patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \
755+ patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent, \
756+ patch('openlp.core.common.Registry') as MockRegistry:
757+ # GIVEN: Mocked out objects and a fake URL
758+ mocked_request_object = MagicMock()
759+ MockRequest.return_value = mocked_request_object
760+ mocked_page_object = MagicMock()
761+ mock_urlopen.return_value = mocked_page_object
762+ mock_get_user_agent.return_value = 'user_agent'
763+ fake_url = 'this://is.a.fake/url'
764+
765+ # WHEN: The get_web_page() method is called
766+ returned_page = get_web_page(fake_url)
767+
768+ # THEN: The correct methods are called with the correct arguments and a web page is returned
769+ MockRequest.assert_called_with(fake_url)
770+ mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent')
771+ self.assertEqual(1, mocked_request_object.add_header.call_count,
772+ 'There should only be 1 call to add_header')
773+ mock_get_user_agent.assert_called_with()
774+ mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
775+ mocked_page_object.geturl.assert_called_with()
776+ self.assertEqual(0, MockRegistry.call_count, 'The Registry() object should have never been called')
777+ self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
778+
779+ def test_get_web_page_with_header(self):
780+ """
781+ Test that adding a header to the call to get_web_page() adds the header to the request
782+ """
783+ with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \
784+ patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \
785+ patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent:
786+ # GIVEN: Mocked out objects, a fake URL and a fake header
787+ mocked_request_object = MagicMock()
788+ MockRequest.return_value = mocked_request_object
789+ mocked_page_object = MagicMock()
790+ mock_urlopen.return_value = mocked_page_object
791+ mock_get_user_agent.return_value = 'user_agent'
792+ fake_url = 'this://is.a.fake/url'
793+ fake_header = ('Fake-Header', 'fake value')
794+
795+ # WHEN: The get_web_page() method is called
796+ returned_page = get_web_page(fake_url, header=fake_header)
797+
798+ # THEN: The correct methods are called with the correct arguments and a web page is returned
799+ MockRequest.assert_called_with(fake_url)
800+ mocked_request_object.add_header.assert_called_with(fake_header[0], fake_header[1])
801+ self.assertEqual(2, mocked_request_object.add_header.call_count,
802+ 'There should only be 2 calls to add_header')
803+ mock_get_user_agent.assert_called_with()
804+ mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
805+ mocked_page_object.geturl.assert_called_with()
806+ self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
807+
808+ def test_get_web_page_with_user_agent_in_headers(self):
809+ """
810+ Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request
811+ """
812+ with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \
813+ patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \
814+ patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent:
815+ # GIVEN: Mocked out objects, a fake URL and a fake header
816+ mocked_request_object = MagicMock()
817+ MockRequest.return_value = mocked_request_object
818+ mocked_page_object = MagicMock()
819+ mock_urlopen.return_value = mocked_page_object
820+ fake_url = 'this://is.a.fake/url'
821+ user_agent_header = ('User-Agent', 'OpenLP/2.2.0')
822+
823+ # WHEN: The get_web_page() method is called
824+ returned_page = get_web_page(fake_url, header=user_agent_header)
825+
826+ # THEN: The correct methods are called with the correct arguments and a web page is returned
827+ MockRequest.assert_called_with(fake_url)
828+ mocked_request_object.add_header.assert_called_with(user_agent_header[0], user_agent_header[1])
829+ self.assertEqual(1, mocked_request_object.add_header.call_count,
830+ 'There should only be 1 call to add_header')
831+ self.assertEqual(0, mock_get_user_agent.call_count, 'get_user_agent should not have been called')
832+ mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
833+ mocked_page_object.geturl.assert_called_with()
834+ self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
835+
836+ def test_get_web_page_update_openlp(self):
837+ """
838+ Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events()
839+ """
840+ with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \
841+ patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \
842+ patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent, \
843+ patch('openlp.core.common.httputils.Registry') as MockRegistry:
844+ # GIVEN: Mocked out objects, a fake URL
845+ mocked_request_object = MagicMock()
846+ MockRequest.return_value = mocked_request_object
847+ mocked_page_object = MagicMock()
848+ mock_urlopen.return_value = mocked_page_object
849+ mock_get_user_agent.return_value = 'user_agent'
850+ mocked_registry_object = MagicMock()
851+ mocked_application_object = MagicMock()
852+ mocked_registry_object.get.return_value = mocked_application_object
853+ MockRegistry.return_value = mocked_registry_object
854+ fake_url = 'this://is.a.fake/url'
855+
856+ # WHEN: The get_web_page() method is called
857+ returned_page = get_web_page(fake_url, update_openlp=True)
858+
859+ # THEN: The correct methods are called with the correct arguments and a web page is returned
860+ MockRequest.assert_called_with(fake_url)
861+ mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent')
862+ self.assertEqual(1, mocked_request_object.add_header.call_count,
863+ 'There should only be 1 call to add_header')
864+ mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
865+ mocked_page_object.geturl.assert_called_with()
866+ mocked_registry_object.get.assert_called_with('application')
867+ mocked_application_object.process_events.assert_called_with()
868+ self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
869+
870+ def test_get_url_file_size(self):
871+ """
872+ Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events()
873+ """
874+ with patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \
875+ patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent:
876+ # GIVEN: Mocked out objects, a fake URL
877+ mocked_page_object = MagicMock()
878+ mock_urlopen.return_value = mocked_page_object
879+ mock_get_user_agent.return_value = 'user_agent'
880+ fake_url = 'this://is.a.fake/url'
881+
882+ # WHEN: The get_url_file_size() method is called
883+ size = get_url_file_size(fake_url)
884+
885+ # THEN: The correct methods are called with the correct arguments and a web page is returned
886+ mock_urlopen.assert_called_with(fake_url, timeout=30)
887+
888+ @patch('openlp.core.ui.firsttimeform.urllib.request.urlopen')
889+ def test_socket_timeout(self, mocked_urlopen):
890+ """
891+ Test socket timeout gets caught
892+ """
893+ # GIVEN: Mocked urlopen to fake a network disconnect in the middle of a download
894+ mocked_urlopen.side_effect = socket.timeout()
895+
896+ # WHEN: Attempt to retrieve a file
897+ url_get_file(MagicMock(), url='http://localhost/test', f_path=self.tempfile)
898+
899+ # THEN: socket.timeout should have been caught
900+ # NOTE: Test is if $tmpdir/tempfile is still there, then test fails since ftw deletes bad downloaded files
901+ self.assertFalse(os.path.exists(self.tempfile), 'FTW url_get_file should have caught socket.timeout')
902
903=== removed file 'tests/functional/openlp_core_lib/test_webpagereader.py'
904--- tests/functional/openlp_core_lib/test_webpagereader.py 2016-05-31 21:40:13 +0000
905+++ tests/functional/openlp_core_lib/test_webpagereader.py 1970-01-01 00:00:00 +0000
906@@ -1,229 +0,0 @@
907-# -*- coding: utf-8 -*-
908-# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
909-
910-###############################################################################
911-# OpenLP - Open Source Lyrics Projection #
912-# --------------------------------------------------------------------------- #
913-# Copyright (c) 2008-2016 OpenLP Developers #
914-# --------------------------------------------------------------------------- #
915-# This program is free software; you can redistribute it and/or modify it #
916-# under the terms of the GNU General Public License as published by the Free #
917-# Software Foundation; version 2 of the License. #
918-# #
919-# This program is distributed in the hope that it will be useful, but WITHOUT #
920-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
921-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
922-# more details. #
923-# #
924-# You should have received a copy of the GNU General Public License along #
925-# with this program; if not, write to the Free Software Foundation, Inc., 59 #
926-# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
927-###############################################################################
928-"""
929-Functional tests to test the AppLocation class and related methods.
930-"""
931-from unittest import TestCase
932-
933-from openlp.core.lib.webpagereader import _get_user_agent, get_web_page
934-
935-from tests.functional import MagicMock, patch
936-
937-
938-class TestUtils(TestCase):
939- """
940- A test suite to test out various methods around the AppLocation class.
941- """
942- def test_get_user_agent_linux(self):
943- """
944- Test that getting a user agent on Linux returns a user agent suitable for Linux
945- """
946- with patch('openlp.core.lib.webpagereader.sys') as mocked_sys:
947-
948- # GIVEN: The system is Linux
949- mocked_sys.platform = 'linux2'
950-
951- # WHEN: We call _get_user_agent()
952- user_agent = _get_user_agent()
953-
954- # THEN: The user agent is a Linux (or ChromeOS) user agent
955- result = 'Linux' in user_agent or 'CrOS' in user_agent
956- self.assertTrue(result, 'The user agent should be a valid Linux user agent')
957-
958- def test_get_user_agent_windows(self):
959- """
960- Test that getting a user agent on Windows returns a user agent suitable for Windows
961- """
962- with patch('openlp.core.lib.webpagereader.sys') as mocked_sys:
963-
964- # GIVEN: The system is Linux
965- mocked_sys.platform = 'win32'
966-
967- # WHEN: We call _get_user_agent()
968- user_agent = _get_user_agent()
969-
970- # THEN: The user agent is a Linux (or ChromeOS) user agent
971- self.assertIn('Windows', user_agent, 'The user agent should be a valid Windows user agent')
972-
973- def test_get_user_agent_macos(self):
974- """
975- Test that getting a user agent on OS X returns a user agent suitable for OS X
976- """
977- with patch('openlp.core.lib.webpagereader.sys') as mocked_sys:
978-
979- # GIVEN: The system is Linux
980- mocked_sys.platform = 'darwin'
981-
982- # WHEN: We call _get_user_agent()
983- user_agent = _get_user_agent()
984-
985- # THEN: The user agent is a Linux (or ChromeOS) user agent
986- self.assertIn('Mac OS X', user_agent, 'The user agent should be a valid OS X user agent')
987-
988- def test_get_user_agent_default(self):
989- """
990- Test that getting a user agent on a non-Linux/Windows/OS X platform returns the default user agent
991- """
992- with patch('openlp.core.lib.webpagereader.sys') as mocked_sys:
993-
994- # GIVEN: The system is Linux
995- mocked_sys.platform = 'freebsd'
996-
997- # WHEN: We call _get_user_agent()
998- user_agent = _get_user_agent()
999-
1000- # THEN: The user agent is a Linux (or ChromeOS) user agent
1001- self.assertIn('NetBSD', user_agent, 'The user agent should be the default user agent')
1002-
1003- def test_get_web_page_no_url(self):
1004- """
1005- Test that sending a URL of None to the get_web_page method returns None
1006- """
1007- # GIVEN: A None url
1008- test_url = None
1009-
1010- # WHEN: We try to get the test URL
1011- result = get_web_page(test_url)
1012-
1013- # THEN: None should be returned
1014- self.assertIsNone(result, 'The return value of get_web_page should be None')
1015-
1016- def test_get_web_page(self):
1017- """
1018- Test that the get_web_page method works correctly
1019- """
1020- with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
1021- patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
1022- patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent, \
1023- patch('openlp.core.common.Registry') as MockRegistry:
1024- # GIVEN: Mocked out objects and a fake URL
1025- mocked_request_object = MagicMock()
1026- MockRequest.return_value = mocked_request_object
1027- mocked_page_object = MagicMock()
1028- mock_urlopen.return_value = mocked_page_object
1029- mock_get_user_agent.return_value = 'user_agent'
1030- fake_url = 'this://is.a.fake/url'
1031-
1032- # WHEN: The get_web_page() method is called
1033- returned_page = get_web_page(fake_url)
1034-
1035- # THEN: The correct methods are called with the correct arguments and a web page is returned
1036- MockRequest.assert_called_with(fake_url)
1037- mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent')
1038- self.assertEqual(1, mocked_request_object.add_header.call_count,
1039- 'There should only be 1 call to add_header')
1040- mock_get_user_agent.assert_called_with()
1041- mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
1042- mocked_page_object.geturl.assert_called_with()
1043- self.assertEqual(0, MockRegistry.call_count, 'The Registry() object should have never been called')
1044- self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
1045-
1046- def test_get_web_page_with_header(self):
1047- """
1048- Test that adding a header to the call to get_web_page() adds the header to the request
1049- """
1050- with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
1051- patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
1052- patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent:
1053- # GIVEN: Mocked out objects, a fake URL and a fake header
1054- mocked_request_object = MagicMock()
1055- MockRequest.return_value = mocked_request_object
1056- mocked_page_object = MagicMock()
1057- mock_urlopen.return_value = mocked_page_object
1058- mock_get_user_agent.return_value = 'user_agent'
1059- fake_url = 'this://is.a.fake/url'
1060- fake_header = ('Fake-Header', 'fake value')
1061-
1062- # WHEN: The get_web_page() method is called
1063- returned_page = get_web_page(fake_url, header=fake_header)
1064-
1065- # THEN: The correct methods are called with the correct arguments and a web page is returned
1066- MockRequest.assert_called_with(fake_url)
1067- mocked_request_object.add_header.assert_called_with(fake_header[0], fake_header[1])
1068- self.assertEqual(2, mocked_request_object.add_header.call_count,
1069- 'There should only be 2 calls to add_header')
1070- mock_get_user_agent.assert_called_with()
1071- mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
1072- mocked_page_object.geturl.assert_called_with()
1073- self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
1074-
1075- def test_get_web_page_with_user_agent_in_headers(self):
1076- """
1077- Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request
1078- """
1079- with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
1080- patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
1081- patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent:
1082- # GIVEN: Mocked out objects, a fake URL and a fake header
1083- mocked_request_object = MagicMock()
1084- MockRequest.return_value = mocked_request_object
1085- mocked_page_object = MagicMock()
1086- mock_urlopen.return_value = mocked_page_object
1087- fake_url = 'this://is.a.fake/url'
1088- user_agent_header = ('User-Agent', 'OpenLP/2.2.0')
1089-
1090- # WHEN: The get_web_page() method is called
1091- returned_page = get_web_page(fake_url, header=user_agent_header)
1092-
1093- # THEN: The correct methods are called with the correct arguments and a web page is returned
1094- MockRequest.assert_called_with(fake_url)
1095- mocked_request_object.add_header.assert_called_with(user_agent_header[0], user_agent_header[1])
1096- self.assertEqual(1, mocked_request_object.add_header.call_count,
1097- 'There should only be 1 call to add_header')
1098- self.assertEqual(0, mock_get_user_agent.call_count, '_get_user_agent should not have been called')
1099- mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
1100- mocked_page_object.geturl.assert_called_with()
1101- self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
1102-
1103- def test_get_web_page_update_openlp(self):
1104- """
1105- Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events()
1106- """
1107- with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
1108- patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
1109- patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent, \
1110- patch('openlp.core.lib.webpagereader.Registry') as MockRegistry:
1111- # GIVEN: Mocked out objects, a fake URL
1112- mocked_request_object = MagicMock()
1113- MockRequest.return_value = mocked_request_object
1114- mocked_page_object = MagicMock()
1115- mock_urlopen.return_value = mocked_page_object
1116- mock_get_user_agent.return_value = 'user_agent'
1117- mocked_registry_object = MagicMock()
1118- mocked_application_object = MagicMock()
1119- mocked_registry_object.get.return_value = mocked_application_object
1120- MockRegistry.return_value = mocked_registry_object
1121- fake_url = 'this://is.a.fake/url'
1122-
1123- # WHEN: The get_web_page() method is called
1124- returned_page = get_web_page(fake_url, update_openlp=True)
1125-
1126- # THEN: The correct methods are called with the correct arguments and a web page is returned
1127- MockRequest.assert_called_with(fake_url)
1128- mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent')
1129- self.assertEqual(1, mocked_request_object.add_header.call_count,
1130- 'There should only be 1 call to add_header')
1131- mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
1132- mocked_page_object.geturl.assert_called_with()
1133- mocked_registry_object.get.assert_called_with('application')
1134- mocked_application_object.process_events.assert_called_with()
1135- self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
1136
1137=== modified file 'tests/functional/openlp_core_ui/test_first_time.py'
1138--- tests/functional/openlp_core_ui/test_first_time.py 2016-05-31 21:40:13 +0000
1139+++ tests/functional/openlp_core_ui/test_first_time.py 2016-12-21 10:20:49 +0000
1140@@ -31,7 +31,7 @@
1141 from tests.functional import patch
1142 from tests.helpers.testmixin import TestMixin
1143
1144-from openlp.core.lib.webpagereader import CONNECTION_RETRIES, get_web_page
1145+from openlp.core.common.httputils import CONNECTION_RETRIES, get_web_page
1146
1147
1148 class TestFirstTimeWizard(TestMixin, TestCase):
1149
1150=== modified file 'tests/functional/openlp_core_ui/test_firsttimeform.py'
1151--- tests/functional/openlp_core_ui/test_firsttimeform.py 2016-05-31 21:40:13 +0000
1152+++ tests/functional/openlp_core_ui/test_firsttimeform.py 2016-12-21 10:20:49 +0000
1153@@ -23,7 +23,6 @@
1154 Package to test the openlp.core.ui.firsttimeform package.
1155 """
1156 import os
1157-import socket
1158 import tempfile
1159 import urllib
1160 from unittest import TestCase
1161@@ -236,20 +235,3 @@
1162 # THEN: the critical_error_message_box should have been called
1163 self.assertEquals(mocked_message_box.mock_calls[1][1][0], 'Network Error 407',
1164 'first_time_form should have caught Network Error')
1165-
1166- @patch('openlp.core.ui.firsttimeform.urllib.request.urlopen')
1167- def test_socket_timeout(self, mocked_urlopen):
1168- """
1169- Test socket timeout gets caught
1170- """
1171- # GIVEN: Mocked urlopen to fake a network disconnect in the middle of a download
1172- first_time_form = FirstTimeForm(None)
1173- first_time_form.initialize(MagicMock())
1174- mocked_urlopen.side_effect = socket.timeout()
1175-
1176- # WHEN: Attempt to retrieve a file
1177- first_time_form.url_get_file(url='http://localhost/test', f_path=self.tempfile)
1178-
1179- # THEN: socket.timeout should have been caught
1180- # NOTE: Test is if $tmpdir/tempfile is still there, then test fails since ftw deletes bad downloaded files
1181- self.assertFalse(os.path.exists(self.tempfile), 'FTW url_get_file should have caught socket.timeout')