Merge lp:~tomasgroth/openlp/song-import-fixes25 into lp:openlp

Proposed by Tomas Groth on 2017-01-20
Status: Merged
Merged at revision: 2718
Proposed branch: lp:~tomasgroth/openlp/song-import-fixes25
Merge into: lp:openlp
Diff against target: 311 lines (+130/-29)
9 files modified
openlp/plugins/songs/lib/__init__.py (+2/-2)
openlp/plugins/songs/lib/importers/easyslides.py (+12/-12)
openlp/plugins/songs/lib/importers/songbeamer.py (+9/-6)
openlp/plugins/songs/lib/importers/videopsalm.py (+4/-2)
tests/functional/openlp_plugins/songs/test_easyslidesimport.py (+2/-0)
tests/resources/easyslidessongs/Amazing Grace.json (+6/-6)
tests/resources/easyslidessongs/Export_2017-01-12_BB.json (+44/-0)
tests/resources/easyslidessongs/Export_2017-01-12_BB.xml (+50/-0)
tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json (+1/-1)
To merge this branch: bzr merge lp:~tomasgroth/openlp/song-import-fixes25
Reviewer Review Type Date Requested Status
Tim Bentley 2017-01-20 Approve on 2017-01-20
Review via email: mp+315270@code.launchpad.net

Description of the change

Clean search lyrics for formatting tags. Fixes bug #1655988.
Fix an issue with easyslide import not handling verse order correctly. Fixes bug #1655985.
Improve the songbeamer encoding detection. Fixes bug #1530597.
Handle a few videopsalm quirks. Fixes bug #1652851.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'openlp/plugins/songs/lib/__init__.py'
2--- openlp/plugins/songs/lib/__init__.py 2016-12-31 11:01:36 +0000
3+++ openlp/plugins/songs/lib/__init__.py 2017-01-20 20:51:35 +0000
4@@ -30,7 +30,7 @@
5 from PyQt5 import QtWidgets
6
7 from openlp.core.common import AppLocation, CONTROL_CHARS
8-from openlp.core.lib import translate
9+from openlp.core.lib import translate, clean_tags
10 from openlp.plugins.songs.lib.db import Author, MediaFile, Song, Topic
11 from openlp.plugins.songs.lib.ui import SongStrings
12
13@@ -380,7 +380,7 @@
14 if isinstance(song.lyrics, bytes):
15 song.lyrics = str(song.lyrics, encoding='utf8')
16 verses = SongXML().get_verses(song.lyrics)
17- song.search_lyrics = ' '.join([clean_string(verse[1]) for verse in verses])
18+ song.search_lyrics = ' '.join([clean_string(clean_tags(verse[1])) for verse in verses])
19 # The song does not have any author, add one.
20 if not song.authors_songs:
21 name = SongStrings.AuthorUnknown
22
23=== modified file 'openlp/plugins/songs/lib/importers/easyslides.py'
24--- openlp/plugins/songs/lib/importers/easyslides.py 2016-12-31 11:01:36 +0000
25+++ openlp/plugins/songs/lib/importers/easyslides.py 2017-01-20 20:51:35 +0000
26@@ -180,7 +180,7 @@
27 reg = default_region
28 verses[reg] = {}
29 # instance differentiates occurrences of same verse tag
30- vt = 'V'
31+ vt = 'v'
32 vn = '1'
33 inst = 1
34 for line in lines:
35@@ -193,14 +193,14 @@
36 inst += 1
37 else:
38 # separators are not used, so empty line starts a new verse
39- vt = 'V'
40+ vt = 'v'
41 vn = len(verses[reg].get(vt, {})) + 1
42 inst = 1
43 elif line[0:7] == '[region':
44 reg = self._extract_region(line)
45 verses.setdefault(reg, {})
46 if not regions_in_verses:
47- vt = 'V'
48+ vt = 'v'
49 vn = '1'
50 inst = 1
51 elif line[0] == '[':
52@@ -213,7 +213,7 @@
53 if match:
54 marker = match.group(1).strip()
55 vn = match.group(2)
56- vt = MarkTypes.get(marker, 'O') if marker else 'V'
57+ vt = MarkTypes.get(marker, 'o') if marker else 'v'
58 if regions_in_verses:
59 region = default_region
60 inst = 1
61@@ -238,13 +238,13 @@
62 lines = '\n'.join(verses[reg][vt][vn][inst])
63 self.add_verse(lines, versetag)
64 SeqTypes = {
65- 'p': 'P1',
66- 'q': 'P2',
67- 'c': 'C1',
68- 't': 'C2',
69- 'b': 'B1',
70- 'w': 'B2',
71- 'e': 'E1'}
72+ 'p': 'p1',
73+ 'q': 'p2',
74+ 'c': 'c1',
75+ 't': 'c2',
76+ 'b': 'b1',
77+ 'w': 'b2',
78+ 'e': 'e1'}
79 # Make use of Sequence data, determining the order of verses
80 try:
81 order = str(song.Sequence).strip().split(',')
82@@ -252,7 +252,7 @@
83 if not tag:
84 continue
85 elif tag[0].isdigit():
86- tag = 'V' + tag
87+ tag = 'v' + tag
88 elif tag.lower() in SeqTypes:
89 tag = SeqTypes[tag.lower()]
90 else:
91
92=== modified file 'openlp/plugins/songs/lib/importers/songbeamer.py'
93--- openlp/plugins/songs/lib/importers/songbeamer.py 2016-12-31 11:01:36 +0000
94+++ openlp/plugins/songs/lib/importers/songbeamer.py 2017-01-20 20:51:35 +0000
95@@ -28,6 +28,7 @@
96 import os
97 import re
98
99+from openlp.core.common import get_file_encoding
100 from openlp.plugins.songs.lib import VerseType
101 from openlp.plugins.songs.lib.importers.songimport import SongImport
102
103@@ -113,13 +114,15 @@
104 read_verses = False
105 file_name = os.path.split(import_file)[1]
106 if os.path.isfile(import_file):
107- # First open in binary mode to detect the encoding
108- detect_file = open(import_file, 'rb')
109- details = chardet.detect(detect_file.read())
110- detect_file.close()
111- infile = codecs.open(import_file, 'r', details['encoding'])
112+ # Detect the encoding
113+ self.input_file_encoding = get_file_encoding(import_file)['encoding']
114+ # The encoding should only be ANSI (cp1252), UTF-8, Unicode, Big-Endian-Unicode.
115+ # So if it doesn't start with 'u' we default to cp1252. See:
116+ # https://forum.songbeamer.com/viewtopic.php?p=419&sid=ca4814924e37c11e4438b7272a98b6f2
117+ if self.input_file_encoding.lower().startswith('u'):
118+ self.input_file_encoding = 'cp1252'
119+ infile = open(import_file, 'rt', encoding=self.input_file_encoding)
120 song_data = infile.readlines()
121- infile.close()
122 else:
123 continue
124 self.title = file_name.split('.sng')[0]
125
126=== modified file 'openlp/plugins/songs/lib/importers/videopsalm.py'
127--- openlp/plugins/songs/lib/importers/videopsalm.py 2016-12-31 11:01:36 +0000
128+++ openlp/plugins/songs/lib/importers/videopsalm.py 2017-01-20 20:51:35 +0000
129@@ -65,8 +65,8 @@
130 if c == '\n':
131 if inside_quotes:
132 processed_content += '\\n'
133- # Put keys in quotes
134- elif c.isalnum() and not inside_quotes:
135+ # Put keys in quotes. The '-' is for handling nagative numbers
136+ elif (c.isalnum() or c == '-') and not inside_quotes:
137 processed_content += '"' + c
138 c = next(file_content_it)
139 while c.isalnum():
140@@ -121,6 +121,8 @@
141 if 'Memo3' in song:
142 self.add_comment(song['Memo3'])
143 for verse in song['Verses']:
144+ if 'Text' not in verse:
145+ continue
146 self.add_verse(verse['Text'], 'v')
147 if not self.finish():
148 self.log_error('Could not import {title}'.format(title=self.title))
149
150=== modified file 'tests/functional/openlp_plugins/songs/test_easyslidesimport.py'
151--- tests/functional/openlp_plugins/songs/test_easyslidesimport.py 2016-12-31 11:01:36 +0000
152+++ tests/functional/openlp_plugins/songs/test_easyslidesimport.py 2017-01-20 20:51:35 +0000
153@@ -43,3 +43,5 @@
154 """
155 self.file_import(os.path.join(TEST_PATH, 'amazing-grace.xml'),
156 self.load_external_result_data(os.path.join(TEST_PATH, 'Amazing Grace.json')))
157+ self.file_import(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.xml'),
158+ self.load_external_result_data(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.json')))
159
160=== modified file 'tests/resources/easyslidessongs/Amazing Grace.json'
161--- tests/resources/easyslidessongs/Amazing Grace.json 2016-01-08 21:42:36 +0000
162+++ tests/resources/easyslidessongs/Amazing Grace.json 2017-01-20 20:51:35 +0000
163@@ -6,27 +6,27 @@
164 "verses": [
165 [
166 "Amazing grace! How sweet the sound\nThat saved a wretch like me;\nI once was lost, but now am found,\nWas blind, but now I see.",
167- "V1"
168+ "v1"
169 ],
170 [
171 "'Twas grace that taught my heart to fear,\nAnd grace my fears relieved;\nHow precious did that grace appear,\nThe hour I first believed!",
172- "V2"
173+ "v2"
174 ],
175 [
176 "Through many dangers, toils and snares\nI have already come;\n'Tis grace that brought me safe thus far,\nAnd grace will lead me home.",
177- "V3"
178+ "v3"
179 ],
180 [
181 "The Lord has promised good to me,\nHis word my hope secures;\nHe will my shield and portion be\nAs long as life endures.",
182- "V4"
183+ "v4"
184 ],
185 [
186 "Yes, when this heart and flesh shall fail,\nAnd mortal life shall cease,\nI shall possess within the veil\nA life of joy and peace.",
187- "V5"
188+ "v5"
189 ],
190 [
191 "When we've been there a thousand years,\nBright shining as the sun,\nWe've no less days to sing God's praise\nThan when we first begun.",
192- "V6"
193+ "v6"
194 ]
195 ]
196 }
197
198=== added file 'tests/resources/easyslidessongs/Export_2017-01-12_BB.json'
199--- tests/resources/easyslidessongs/Export_2017-01-12_BB.json 1970-01-01 00:00:00 +0000
200+++ tests/resources/easyslidessongs/Export_2017-01-12_BB.json 2017-01-20 20:51:35 +0000
201@@ -0,0 +1,44 @@
202+{
203+ "title": "BBBBBBBBB",
204+ "authors": [
205+ "John Newton (1725-1807)"
206+ ],
207+ "verses": [
208+ [
209+ "V1V1V1V1V1V1\nV1V1V1V1V1V1",
210+ "v1"
211+ ],
212+ [
213+ "V2V2V2V2V2V2\nV2V2V2V2V2V2",
214+ "v2"
215+ ],
216+ [
217+ "C1C1C1C1C1C1\nC1C1C1C1C1C1",
218+ "c1"
219+ ],
220+ [
221+ "C2C2C2C2C2C2\nC2C2C2C2C2C2",
222+ "c2"
223+ ],
224+ [
225+ "B1B1B1B1B1B1\nB1B1B1B1B1B1",
226+ "b1"
227+ ],
228+ [
229+ "B2B2B2B2B2B2\nB2B2B2B2B2B2",
230+ "b2"
231+ ],
232+ [
233+ "PRE1PRE1PRE1\nPRE1PRE1PRE1",
234+ "p1"
235+ ],
236+ [
237+ "PRE2PRE2PRE2\nPRE2PRE2PRE2",
238+ "p2"
239+ ],
240+ [
241+ "ENDENDENDEND\nENDENDENDEND",
242+ "e1"
243+ ]
244+ ]
245+}
246
247=== added file 'tests/resources/easyslidessongs/Export_2017-01-12_BB.xml'
248--- tests/resources/easyslidessongs/Export_2017-01-12_BB.xml 1970-01-01 00:00:00 +0000
249+++ tests/resources/easyslidessongs/Export_2017-01-12_BB.xml 2017-01-20 20:51:35 +0000
250@@ -0,0 +1,50 @@
251+<?xml version="1.0" encoding="utf-8"?>
252+<EasiSlides>
253+ <Item>
254+ <Title1>BBBBBBBBB</Title1>
255+ <Title2 />
256+ <Folder>NAGY</Folder>
257+ <SongNumber>0</SongNumber>
258+ <Contents>[1]
259+V1V1V1V1V1V1
260+V1V1V1V1V1V1
261+[2]
262+V2V2V2V2V2V2
263+V2V2V2V2V2V2
264+[chorus]
265+C1C1C1C1C1C1
266+C1C1C1C1C1C1
267+[chorus 2]
268+C2C2C2C2C2C2
269+C2C2C2C2C2C2
270+[bridge]
271+B1B1B1B1B1B1
272+B1B1B1B1B1B1
273+[bridge 2]
274+B2B2B2B2B2B2
275+B2B2B2B2B2B2
276+[prechorus]
277+PRE1PRE1PRE1
278+PRE1PRE1PRE1
279+[prechorus 2]
280+PRE2PRE2PRE2
281+PRE2PRE2PRE2
282+[ending]
283+ENDENDENDEND
284+ENDENDENDEND</Contents>
285+ <Notations />
286+ <Sequence>1,2,c,t,b,w,p,q,e</Sequence>
287+ <Writer />
288+ <Copyright />
289+ <Category />
290+ <Timing />
291+ <MusicKey />
292+ <Capo>-1</Capo>
293+ <LicenceAdmin1 />
294+ <LicenceAdmin2 />
295+ <BookReference />
296+ <UserReference />
297+ <FormatData />
298+ <Settings>10=&gt;</Settings>
299+ </Item>
300+</EasiSlides>
301\ No newline at end of file
302
303=== modified file 'tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json'
304--- tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json 2015-12-17 21:39:52 +0000
305+++ tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json 2017-01-20 20:51:35 +0000
306@@ -1,4 +1,4 @@
307-{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Copyright:"Public
308+{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Capo:-1,Copyright:"Public
309 Domain",Theme:"tema1
310 tema2",CCLI:"12345",Alias:"A safe stronghold",Memo1:"This is
311 the first comment