Merge lp:~tomasgroth/openlp/25bugfixes3 into lp:openlp

Proposed by Tomas Groth on 2016-06-15
Status: Merged
Merged at revision: 2677
Proposed branch: lp:~tomasgroth/openlp/25bugfixes3
Merge into: lp:openlp
Diff against target: 263 lines (+81/-24)
9 files modified
openlp/plugins/songs/lib/importers/mediashout.py (+18/-11)
openlp/plugins/songs/lib/importers/opspro.py (+3/-3)
openlp/plugins/songs/lib/importers/presentationmanager.py (+7/-1)
openlp/plugins/songs/lib/importers/songpro.py (+1/-1)
openlp/plugins/songs/lib/importers/songshowplus.py (+10/-4)
openlp/plugins/songs/lib/importers/videopsalm.py (+1/-3)
openlp/plugins/songs/lib/importers/worshipcenterpro.py (+1/-1)
tests/functional/openlp_plugins/songs/test_songshowplusimport.py (+2/-0)
tests/resources/songshowplussongs/cleanse-me.json (+38/-0)
To merge this branch: bzr merge lp:~tomasgroth/openlp/25bugfixes3
Reviewer Review Type Date Requested Status
Raoul Snyman 2016-06-15 Approve on 2016-06-15
Review via email: mp+297541@code.launchpad.net

Description of the change

Fix various pyodbc related issues. Fixes bug 1590657.
Fix of tracback during SongPro import. Fixes bug 1582152.
Fix traceback during songshowplus import. Fixes bug 1585489.
Skip PresentationManager files we do not support.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'openlp/plugins/songs/lib/importers/mediashout.py'
2--- openlp/plugins/songs/lib/importers/mediashout.py 2016-05-27 08:13:14 +0000
3+++ openlp/plugins/songs/lib/importers/mediashout.py 2016-06-15 20:11:03 +0000
4@@ -24,15 +24,17 @@
5 a MediaShout database into the OpenLP database.
6 """
7
8-# WARNING: See https://docs.python.org/2/library/sqlite3.html for value substitution
9+# WARNING: See https://docs.python.org/3/library/sqlite3.html for value substitution
10 # in SQL statements
11
12 import pyodbc
13+import logging
14
15 from openlp.core.lib import translate
16 from openlp.plugins.songs.lib.importers.songimport import SongImport
17
18 VERSE_TAGS = ['V', 'C', 'B', 'O', 'P', 'I', 'E']
19+log = logging.getLogger(__name__)
20
21
22 class MediaShoutImport(SongImport):
23@@ -44,17 +46,18 @@
24 """
25 Initialise the MediaShout importer.
26 """
27- SongImport.__init__(self, manager, **kwargs)
28+ super(MediaShoutImport, self).__init__(manager, **kwargs)
29
30 def do_import(self):
31 """
32 Receive a single file to import.
33 """
34 try:
35- conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ={source};'
36- 'PWD=6NOZ4eHK7k'.format(sorce=self.import_source))
37- except:
38+ conn = pyodbc.connect('DRIVER={{Microsoft Access Driver (*.mdb)}};DBQ={source};'
39+ 'PWD=6NOZ4eHK7k'.format(source=self.import_source))
40+ except Exception as e:
41 # Unfortunately no specific exception type
42+ log.exception(e)
43 self.log_error(self.import_source, translate('SongsPlugin.MediaShoutImport',
44 'Unable to open the MediaShout database.'))
45 return
46@@ -63,17 +66,21 @@
47 songs = cursor.fetchall()
48 self.import_wizard.progress_bar.setMaximum(len(songs))
49 for song in songs:
50+ topics = []
51 if self.stop_import_flag:
52 break
53- cursor.execute('SELECT Type, Number, Text FROM Verses WHERE Record = ? ORDER BY Type, Number', song.Record)
54+ cursor.execute('SELECT Type, Number, Text FROM Verses WHERE Record = ? ORDER BY Type, Number',
55+ float(song.Record))
56 verses = cursor.fetchall()
57- cursor.execute('SELECT Type, Number, POrder FROM PlayOrder WHERE Record = ? ORDER BY POrder', song.Record)
58+ cursor.execute('SELECT Type, Number, POrder FROM PlayOrder WHERE Record = ? ORDER BY POrder',
59+ float(song.Record))
60 verse_order = cursor.fetchall()
61- cursor.execute('SELECT Name FROM Themes INNER JOIN SongThemes ON SongThemes.ThemeId = Themes.ThemeId '
62- 'WHERE SongThemes.Record = ?', song.Record)
63- topics = cursor.fetchall()
64+ if cursor.tables(table='TableName', tableType='TABLE').fetchone():
65+ cursor.execute('SELECT Name FROM Themes INNER JOIN SongThemes ON SongThemes.ThemeId = Themes.ThemeId '
66+ 'WHERE SongThemes.Record = ?', float(song.Record))
67+ topics = cursor.fetchall()
68 cursor.execute('SELECT Name FROM Groups INNER JOIN SongGroups ON SongGroups.GroupId = Groups.GroupId '
69- 'WHERE SongGroups.Record = ?', song.Record)
70+ 'WHERE SongGroups.Record = ?', float(song.Record))
71 topics += cursor.fetchall()
72 self.process_song(song, verses, verse_order, topics)
73
74
75=== modified file 'openlp/plugins/songs/lib/importers/opspro.py'
76--- openlp/plugins/songs/lib/importers/opspro.py 2016-05-27 08:13:14 +0000
77+++ openlp/plugins/songs/lib/importers/opspro.py 2016-06-15 20:11:03 +0000
78@@ -55,7 +55,7 @@
79 """
80 password = self.extract_mdb_password()
81 try:
82- conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ={source};'
83+ conn = pyodbc.connect('DRIVER={{Microsoft Access Driver (*.mdb)}};DBQ={source};'
84 'PWD={password}'.format(source=self.import_source, password=password))
85 except (pyodbc.DatabaseError, pyodbc.IntegrityError, pyodbc.InternalError, pyodbc.OperationalError) as e:
86 log.warning('Unable to connect the OPS Pro database {source}. {error}'.format(source=self.import_source,
87@@ -74,11 +74,11 @@
88 break
89 # Type means: 0=Original, 1=Projection, 2=Own
90 cursor.execute('SELECT Lyrics, Type, IsDualLanguage FROM Lyrics WHERE SongID = ? AND Type < 2 '
91- 'ORDER BY Type DESC', song.ID)
92+ 'ORDER BY Type DESC', float(song.ID))
93 lyrics = cursor.fetchone()
94 cursor.execute('SELECT CategoryName FROM Category INNER JOIN SongCategory '
95 'ON Category.ID = SongCategory.CategoryID WHERE SongCategory.SongID = ? '
96- 'ORDER BY CategoryName', song.ID)
97+ 'ORDER BY CategoryName', float(song.ID))
98 topics = cursor.fetchall()
99 try:
100 self.process_song(song, lyrics, topics)
101
102=== modified file 'openlp/plugins/songs/lib/importers/presentationmanager.py'
103--- openlp/plugins/songs/lib/importers/presentationmanager.py 2016-05-27 08:13:14 +0000
104+++ openlp/plugins/songs/lib/importers/presentationmanager.py 2016-06-15 20:11:03 +0000
105@@ -56,7 +56,13 @@
106 # Open file with detected encoding and remove encoding declaration
107 text = open(file_path, mode='r', encoding=encoding).read()
108 text = re.sub('.+\?>\n', '', text)
109- tree = etree.fromstring(text, parser=etree.XMLParser(recover=True))
110+ try:
111+ tree = etree.fromstring(text, parser=etree.XMLParser(recover=True))
112+ except ValueError:
113+ self.log_error(file_path,
114+ translate('SongsPlugin.PresentationManagerImport',
115+ 'File is not in XML-format, which is the only format supported.'))
116+ continue
117 root = objectify.fromstring(etree.tostring(tree))
118 self.process_song(root)
119
120
121=== modified file 'openlp/plugins/songs/lib/importers/songpro.py'
122--- openlp/plugins/songs/lib/importers/songpro.py 2015-12-31 22:46:06 +0000
123+++ openlp/plugins/songs/lib/importers/songpro.py 2016-06-15 20:11:03 +0000
124@@ -72,7 +72,7 @@
125 Receive a single file or a list of files to import.
126 """
127 self.encoding = None
128- with open(self.import_source, 'rt') as songs_file:
129+ with open(self.import_source, 'rt', errors='ignore') as songs_file:
130 self.import_wizard.progress_bar.setMaximum(0)
131 tag = ''
132 text = ''
133
134=== modified file 'openlp/plugins/songs/lib/importers/songshowplus.py'
135--- openlp/plugins/songs/lib/importers/songshowplus.py 2016-05-27 08:13:14 +0000
136+++ openlp/plugins/songs/lib/importers/songshowplus.py 2016-06-15 20:11:03 +0000
137@@ -106,6 +106,7 @@
138 song_data = open(file, 'rb')
139 while True:
140 block_key, = struct.unpack("I", song_data.read(4))
141+ log.debug('block_key: %d' % block_key)
142 # The file ends with 4 NULL's
143 if block_key == 0:
144 break
145@@ -117,7 +118,13 @@
146 null, verse_name_length, = struct.unpack("BB", song_data.read(2))
147 verse_name = self.decode(song_data.read(verse_name_length))
148 length_descriptor_size, = struct.unpack("B", song_data.read(1))
149- log.debug(length_descriptor_size)
150+ log.debug('length_descriptor_size: %d' % length_descriptor_size)
151+ # In the case of song_numbers the number is in the data from the
152+ # current position to the next block starts
153+ if block_key == SONG_NUMBER:
154+ sn_bytes = song_data.read(length_descriptor_size - 1)
155+ self.song_number = int.from_bytes(sn_bytes, byteorder='little')
156+ continue
157 # Detect if/how long the length descriptor is
158 if length_descriptor_size == 12 or length_descriptor_size == 20:
159 length_descriptor, = struct.unpack("I", song_data.read(4))
160@@ -127,8 +134,9 @@
161 length_descriptor = 0
162 else:
163 length_descriptor, = struct.unpack("B", song_data.read(1))
164- log.debug(length_descriptor_size)
165+ log.debug('length_descriptor: %d' % length_descriptor)
166 data = song_data.read(length_descriptor)
167+ log.debug(data)
168 if block_key == TITLE:
169 self.title = self.decode(data)
170 elif block_key == AUTHOR:
171@@ -168,8 +176,6 @@
172 self.ssp_verse_order_list.append(verse_tag)
173 elif block_key == SONG_BOOK:
174 self.song_book_name = self.decode(data)
175- elif block_key == SONG_NUMBER:
176- self.song_number = ord(data)
177 elif block_key == CUSTOM_VERSE:
178 verse_tag = self.to_openlp_verse_tag(verse_name)
179 self.add_verse(self.decode(data), verse_tag)
180
181=== modified file 'openlp/plugins/songs/lib/importers/videopsalm.py'
182--- openlp/plugins/songs/lib/importers/videopsalm.py 2016-05-27 08:13:14 +0000
183+++ openlp/plugins/songs/lib/importers/videopsalm.py 2016-06-15 20:11:03 +0000
184@@ -117,6 +117,4 @@
185 if not self.finish():
186 self.log_error('Could not import {title}'.format(title=self.title))
187 except Exception as e:
188- self.log_error(translate('SongsPlugin.VideoPsalmImport', 'File {name}').format(name=file.name),
189- translate('SongsPlugin.VideoPsalmImport', 'Error: {error}').format(error=e))
190- song_file.close()
191+ self.log_error(song_file.name, translate('SongsPlugin.VideoPsalmImport', 'Error: {error}').format(error=e))
192
193=== modified file 'openlp/plugins/songs/lib/importers/worshipcenterpro.py'
194--- openlp/plugins/songs/lib/importers/worshipcenterpro.py 2016-05-27 08:13:14 +0000
195+++ openlp/plugins/songs/lib/importers/worshipcenterpro.py 2016-06-15 20:11:03 +0000
196@@ -49,7 +49,7 @@
197 Receive a single file to import.
198 """
199 try:
200- conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};'
201+ conn = pyodbc.connect('DRIVER={{Microsoft Access Driver (*.mdb)}};'
202 'DBQ={source}'.format(source=self.import_source))
203 except (pyodbc.DatabaseError, pyodbc.IntegrityError, pyodbc.InternalError, pyodbc.OperationalError) as e:
204 log.warning('Unable to connect the WorshipCenter Pro '
205
206=== modified file 'tests/functional/openlp_plugins/songs/test_songshowplusimport.py'
207--- tests/functional/openlp_plugins/songs/test_songshowplusimport.py 2016-05-31 21:40:13 +0000
208+++ tests/functional/openlp_plugins/songs/test_songshowplusimport.py 2016-06-15 20:11:03 +0000
209@@ -52,6 +52,8 @@
210 self.load_external_result_data(os.path.join(TEST_PATH, 'Beautiful Garden Of Prayer.json')))
211 self.file_import([os.path.join(TEST_PATH, 'a mighty fortress is our god.sbsong')],
212 self.load_external_result_data(os.path.join(TEST_PATH, 'a mighty fortress is our god.json')))
213+ self.file_import([os.path.join(TEST_PATH, 'cleanse-me.sbsong')],
214+ self.load_external_result_data(os.path.join(TEST_PATH, 'cleanse-me.json')))
215
216
217 class TestSongShowPlusImport(TestCase):
218
219=== added file 'tests/resources/songshowplussongs/cleanse-me.json'
220--- tests/resources/songshowplussongs/cleanse-me.json 1970-01-01 00:00:00 +0000
221+++ tests/resources/songshowplussongs/cleanse-me.json 2016-06-15 20:11:03 +0000
222@@ -0,0 +1,38 @@
223+{
224+ "authors": [
225+ "J. Edwin Orr"
226+ ],
227+ "ccli_number": 56307,
228+ "comments": "",
229+ "copyright": "Public Domain ",
230+ "song_book_name": "",
231+ "song_number": 438,
232+ "title": "Cleanse Me [438]",
233+ "topics": [
234+ "Cleansing",
235+ "Communion",
236+ "Consecration",
237+ "Holiness",
238+ "Holy Spirit",
239+ "Revival"
240+ ],
241+ "verse_order_list": [],
242+ "verses": [
243+ [
244+ "Search me, O God,\r\nAnd know my heart today;\r\nTry me, O Savior,\r\nKnow my thoughts, I pray.\r\nSee if there be\r\nSome wicked way in me;\r\nCleanse me from every sin\r\nAnd set me free.",
245+ "v1"
246+ ],
247+ [
248+ "I praise Thee, Lord,\r\nFor cleansing me from sin;\r\nFulfill Thy Word,\r\nAnd make me pure within.\r\nFill me with fire\r\nWhere once I burned with shame;\r\nGrant my desire\r\nTo magnify Thy name.",
249+ "v2"
250+ ],
251+ [
252+ "Lord, take my life,\r\nAnd make it wholly Thine;\r\nFill my poor heart\r\nWith Thy great love divine.\r\nTake all my will,\r\nMy passion, self and pride;\r\nI now surrender, Lord\r\nIn me abide.",
253+ "v3"
254+ ],
255+ [
256+ "O Holy Ghost,\r\nRevival comes from Thee;\r\nSend a revival,\r\nStart the work in me.\r\nThy Word declares\r\nThou wilt supply our need;\r\nFor blessings now,\r\nO Lord, I humbly plead.",
257+ "v4"
258+ ]
259+ ]
260+}
261
262=== added file 'tests/resources/songshowplussongs/cleanse-me.sbsong'
263Binary files tests/resources/songshowplussongs/cleanse-me.sbsong 1970-01-01 00:00:00 +0000 and tests/resources/songshowplussongs/cleanse-me.sbsong 2016-06-15 20:11:03 +0000 differ