Merge lp:~tomasgroth/openlp/song-import-fixes25 into lp:openlp

Proposed by Tomas Groth
Status: Merged
Merged at revision: 2718
Proposed branch: lp:~tomasgroth/openlp/song-import-fixes25
Merge into: lp:openlp
Diff against target: 311 lines (+130/-29)
9 files modified
openlp/plugins/songs/lib/__init__.py (+2/-2)
openlp/plugins/songs/lib/importers/easyslides.py (+12/-12)
openlp/plugins/songs/lib/importers/songbeamer.py (+9/-6)
openlp/plugins/songs/lib/importers/videopsalm.py (+4/-2)
tests/functional/openlp_plugins/songs/test_easyslidesimport.py (+2/-0)
tests/resources/easyslidessongs/Amazing Grace.json (+6/-6)
tests/resources/easyslidessongs/Export_2017-01-12_BB.json (+44/-0)
tests/resources/easyslidessongs/Export_2017-01-12_BB.xml (+50/-0)
tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json (+1/-1)
To merge this branch: bzr merge lp:~tomasgroth/openlp/song-import-fixes25
Reviewer Review Type Date Requested Status
Tim Bentley Approve
Review via email: mp+315270@code.launchpad.net

Description of the change

Clean search lyrics for formatting tags. Fixes bug #1655988.
Fix an issue with easyslide import not handling verse order correctly. Fixes bug #1655985.
Improve the songbeamer encoding detection. Fixes bug #1530597.
Handle a few videopsalm quirks. Fixes bug #1652851.

To post a comment you must log in.
Revision history for this message
Tomas Groth (tomasgroth) wrote :
Revision history for this message
Tim Bentley (trb143) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'openlp/plugins/songs/lib/__init__.py'
--- openlp/plugins/songs/lib/__init__.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/__init__.py 2017-01-20 20:51:35 +0000
@@ -30,7 +30,7 @@
30from PyQt5 import QtWidgets30from PyQt5 import QtWidgets
3131
32from openlp.core.common import AppLocation, CONTROL_CHARS32from openlp.core.common import AppLocation, CONTROL_CHARS
33from openlp.core.lib import translate33from openlp.core.lib import translate, clean_tags
34from openlp.plugins.songs.lib.db import Author, MediaFile, Song, Topic34from openlp.plugins.songs.lib.db import Author, MediaFile, Song, Topic
35from openlp.plugins.songs.lib.ui import SongStrings35from openlp.plugins.songs.lib.ui import SongStrings
3636
@@ -380,7 +380,7 @@
380 if isinstance(song.lyrics, bytes):380 if isinstance(song.lyrics, bytes):
381 song.lyrics = str(song.lyrics, encoding='utf8')381 song.lyrics = str(song.lyrics, encoding='utf8')
382 verses = SongXML().get_verses(song.lyrics)382 verses = SongXML().get_verses(song.lyrics)
383 song.search_lyrics = ' '.join([clean_string(verse[1]) for verse in verses])383 song.search_lyrics = ' '.join([clean_string(clean_tags(verse[1])) for verse in verses])
384 # The song does not have any author, add one.384 # The song does not have any author, add one.
385 if not song.authors_songs:385 if not song.authors_songs:
386 name = SongStrings.AuthorUnknown386 name = SongStrings.AuthorUnknown
387387
=== modified file 'openlp/plugins/songs/lib/importers/easyslides.py'
--- openlp/plugins/songs/lib/importers/easyslides.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/easyslides.py 2017-01-20 20:51:35 +0000
@@ -180,7 +180,7 @@
180 reg = default_region180 reg = default_region
181 verses[reg] = {}181 verses[reg] = {}
182 # instance differentiates occurrences of same verse tag182 # instance differentiates occurrences of same verse tag
183 vt = 'V'183 vt = 'v'
184 vn = '1'184 vn = '1'
185 inst = 1185 inst = 1
186 for line in lines:186 for line in lines:
@@ -193,14 +193,14 @@
193 inst += 1193 inst += 1
194 else:194 else:
195 # separators are not used, so empty line starts a new verse195 # separators are not used, so empty line starts a new verse
196 vt = 'V'196 vt = 'v'
197 vn = len(verses[reg].get(vt, {})) + 1197 vn = len(verses[reg].get(vt, {})) + 1
198 inst = 1198 inst = 1
199 elif line[0:7] == '[region':199 elif line[0:7] == '[region':
200 reg = self._extract_region(line)200 reg = self._extract_region(line)
201 verses.setdefault(reg, {})201 verses.setdefault(reg, {})
202 if not regions_in_verses:202 if not regions_in_verses:
203 vt = 'V'203 vt = 'v'
204 vn = '1'204 vn = '1'
205 inst = 1205 inst = 1
206 elif line[0] == '[':206 elif line[0] == '[':
@@ -213,7 +213,7 @@
213 if match:213 if match:
214 marker = match.group(1).strip()214 marker = match.group(1).strip()
215 vn = match.group(2)215 vn = match.group(2)
216 vt = MarkTypes.get(marker, 'O') if marker else 'V'216 vt = MarkTypes.get(marker, 'o') if marker else 'v'
217 if regions_in_verses:217 if regions_in_verses:
218 region = default_region218 region = default_region
219 inst = 1219 inst = 1
@@ -238,13 +238,13 @@
238 lines = '\n'.join(verses[reg][vt][vn][inst])238 lines = '\n'.join(verses[reg][vt][vn][inst])
239 self.add_verse(lines, versetag)239 self.add_verse(lines, versetag)
240 SeqTypes = {240 SeqTypes = {
241 'p': 'P1',241 'p': 'p1',
242 'q': 'P2',242 'q': 'p2',
243 'c': 'C1',243 'c': 'c1',
244 't': 'C2',244 't': 'c2',
245 'b': 'B1',245 'b': 'b1',
246 'w': 'B2',246 'w': 'b2',
247 'e': 'E1'}247 'e': 'e1'}
248 # Make use of Sequence data, determining the order of verses248 # Make use of Sequence data, determining the order of verses
249 try:249 try:
250 order = str(song.Sequence).strip().split(',')250 order = str(song.Sequence).strip().split(',')
@@ -252,7 +252,7 @@
252 if not tag:252 if not tag:
253 continue253 continue
254 elif tag[0].isdigit():254 elif tag[0].isdigit():
255 tag = 'V' + tag255 tag = 'v' + tag
256 elif tag.lower() in SeqTypes:256 elif tag.lower() in SeqTypes:
257 tag = SeqTypes[tag.lower()]257 tag = SeqTypes[tag.lower()]
258 else:258 else:
259259
=== modified file 'openlp/plugins/songs/lib/importers/songbeamer.py'
--- openlp/plugins/songs/lib/importers/songbeamer.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/songbeamer.py 2017-01-20 20:51:35 +0000
@@ -28,6 +28,7 @@
28import os28import os
29import re29import re
3030
31from openlp.core.common import get_file_encoding
31from openlp.plugins.songs.lib import VerseType32from openlp.plugins.songs.lib import VerseType
32from openlp.plugins.songs.lib.importers.songimport import SongImport33from openlp.plugins.songs.lib.importers.songimport import SongImport
3334
@@ -113,13 +114,15 @@
113 read_verses = False114 read_verses = False
114 file_name = os.path.split(import_file)[1]115 file_name = os.path.split(import_file)[1]
115 if os.path.isfile(import_file):116 if os.path.isfile(import_file):
116 # First open in binary mode to detect the encoding117 # Detect the encoding
117 detect_file = open(import_file, 'rb')118 self.input_file_encoding = get_file_encoding(import_file)['encoding']
118 details = chardet.detect(detect_file.read())119 # The encoding should only be ANSI (cp1252), UTF-8, Unicode, Big-Endian-Unicode.
119 detect_file.close()120 # So if it doesn't start with 'u' we default to cp1252. See:
120 infile = codecs.open(import_file, 'r', details['encoding'])121 # https://forum.songbeamer.com/viewtopic.php?p=419&sid=ca4814924e37c11e4438b7272a98b6f2
122 if self.input_file_encoding.lower().startswith('u'):
123 self.input_file_encoding = 'cp1252'
124 infile = open(import_file, 'rt', encoding=self.input_file_encoding)
121 song_data = infile.readlines()125 song_data = infile.readlines()
122 infile.close()
123 else:126 else:
124 continue127 continue
125 self.title = file_name.split('.sng')[0]128 self.title = file_name.split('.sng')[0]
126129
=== modified file 'openlp/plugins/songs/lib/importers/videopsalm.py'
--- openlp/plugins/songs/lib/importers/videopsalm.py 2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/videopsalm.py 2017-01-20 20:51:35 +0000
@@ -65,8 +65,8 @@
65 if c == '\n':65 if c == '\n':
66 if inside_quotes:66 if inside_quotes:
67 processed_content += '\\n'67 processed_content += '\\n'
68 # Put keys in quotes68 # Put keys in quotes. The '-' is for handling nagative numbers
69 elif c.isalnum() and not inside_quotes:69 elif (c.isalnum() or c == '-') and not inside_quotes:
70 processed_content += '"' + c70 processed_content += '"' + c
71 c = next(file_content_it)71 c = next(file_content_it)
72 while c.isalnum():72 while c.isalnum():
@@ -121,6 +121,8 @@
121 if 'Memo3' in song:121 if 'Memo3' in song:
122 self.add_comment(song['Memo3'])122 self.add_comment(song['Memo3'])
123 for verse in song['Verses']:123 for verse in song['Verses']:
124 if 'Text' not in verse:
125 continue
124 self.add_verse(verse['Text'], 'v')126 self.add_verse(verse['Text'], 'v')
125 if not self.finish():127 if not self.finish():
126 self.log_error('Could not import {title}'.format(title=self.title))128 self.log_error('Could not import {title}'.format(title=self.title))
127129
=== modified file 'tests/functional/openlp_plugins/songs/test_easyslidesimport.py'
--- tests/functional/openlp_plugins/songs/test_easyslidesimport.py 2016-12-31 11:01:36 +0000
+++ tests/functional/openlp_plugins/songs/test_easyslidesimport.py 2017-01-20 20:51:35 +0000
@@ -43,3 +43,5 @@
43 """43 """
44 self.file_import(os.path.join(TEST_PATH, 'amazing-grace.xml'),44 self.file_import(os.path.join(TEST_PATH, 'amazing-grace.xml'),
45 self.load_external_result_data(os.path.join(TEST_PATH, 'Amazing Grace.json')))45 self.load_external_result_data(os.path.join(TEST_PATH, 'Amazing Grace.json')))
46 self.file_import(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.xml'),
47 self.load_external_result_data(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.json')))
4648
=== modified file 'tests/resources/easyslidessongs/Amazing Grace.json'
--- tests/resources/easyslidessongs/Amazing Grace.json 2016-01-08 21:42:36 +0000
+++ tests/resources/easyslidessongs/Amazing Grace.json 2017-01-20 20:51:35 +0000
@@ -6,27 +6,27 @@
6 "verses": [6 "verses": [
7 [7 [
8 "Amazing grace! How sweet the sound\nThat saved a wretch like me;\nI once was lost, but now am found,\nWas blind, but now I see.",8 "Amazing grace! How sweet the sound\nThat saved a wretch like me;\nI once was lost, but now am found,\nWas blind, but now I see.",
9 "V1"9 "v1"
10 ],10 ],
11 [11 [
12 "'Twas grace that taught my heart to fear,\nAnd grace my fears relieved;\nHow precious did that grace appear,\nThe hour I first believed!",12 "'Twas grace that taught my heart to fear,\nAnd grace my fears relieved;\nHow precious did that grace appear,\nThe hour I first believed!",
13 "V2"13 "v2"
14 ],14 ],
15 [15 [
16 "Through many dangers, toils and snares\nI have already come;\n'Tis grace that brought me safe thus far,\nAnd grace will lead me home.",16 "Through many dangers, toils and snares\nI have already come;\n'Tis grace that brought me safe thus far,\nAnd grace will lead me home.",
17 "V3"17 "v3"
18 ],18 ],
19 [19 [
20 "The Lord has promised good to me,\nHis word my hope secures;\nHe will my shield and portion be\nAs long as life endures.",20 "The Lord has promised good to me,\nHis word my hope secures;\nHe will my shield and portion be\nAs long as life endures.",
21 "V4"21 "v4"
22 ],22 ],
23 [23 [
24 "Yes, when this heart and flesh shall fail,\nAnd mortal life shall cease,\nI shall possess within the veil\nA life of joy and peace.",24 "Yes, when this heart and flesh shall fail,\nAnd mortal life shall cease,\nI shall possess within the veil\nA life of joy and peace.",
25 "V5"25 "v5"
26 ],26 ],
27 [27 [
28 "When we've been there a thousand years,\nBright shining as the sun,\nWe've no less days to sing God's praise\nThan when we first begun.",28 "When we've been there a thousand years,\nBright shining as the sun,\nWe've no less days to sing God's praise\nThan when we first begun.",
29 "V6"29 "v6"
30 ]30 ]
31 ]31 ]
32}32}
3333
=== added file 'tests/resources/easyslidessongs/Export_2017-01-12_BB.json'
--- tests/resources/easyslidessongs/Export_2017-01-12_BB.json 1970-01-01 00:00:00 +0000
+++ tests/resources/easyslidessongs/Export_2017-01-12_BB.json 2017-01-20 20:51:35 +0000
@@ -0,0 +1,44 @@
1{
2 "title": "BBBBBBBBB",
3 "authors": [
4 "John Newton (1725-1807)"
5 ],
6 "verses": [
7 [
8 "V1V1V1V1V1V1\nV1V1V1V1V1V1",
9 "v1"
10 ],
11 [
12 "V2V2V2V2V2V2\nV2V2V2V2V2V2",
13 "v2"
14 ],
15 [
16 "C1C1C1C1C1C1\nC1C1C1C1C1C1",
17 "c1"
18 ],
19 [
20 "C2C2C2C2C2C2\nC2C2C2C2C2C2",
21 "c2"
22 ],
23 [
24 "B1B1B1B1B1B1\nB1B1B1B1B1B1",
25 "b1"
26 ],
27 [
28 "B2B2B2B2B2B2\nB2B2B2B2B2B2",
29 "b2"
30 ],
31 [
32 "PRE1PRE1PRE1\nPRE1PRE1PRE1",
33 "p1"
34 ],
35 [
36 "PRE2PRE2PRE2\nPRE2PRE2PRE2",
37 "p2"
38 ],
39 [
40 "ENDENDENDEND\nENDENDENDEND",
41 "e1"
42 ]
43 ]
44}
045
=== added file 'tests/resources/easyslidessongs/Export_2017-01-12_BB.xml'
--- tests/resources/easyslidessongs/Export_2017-01-12_BB.xml 1970-01-01 00:00:00 +0000
+++ tests/resources/easyslidessongs/Export_2017-01-12_BB.xml 2017-01-20 20:51:35 +0000
@@ -0,0 +1,50 @@
1<?xml version="1.0" encoding="utf-8"?>
2<EasiSlides>
3 <Item>
4 <Title1>BBBBBBBBB</Title1>
5 <Title2 />
6 <Folder>NAGY</Folder>
7 <SongNumber>0</SongNumber>
8 <Contents>[1]
9V1V1V1V1V1V1
10V1V1V1V1V1V1
11[2]
12V2V2V2V2V2V2
13V2V2V2V2V2V2
14[chorus]
15C1C1C1C1C1C1
16C1C1C1C1C1C1
17[chorus 2]
18C2C2C2C2C2C2
19C2C2C2C2C2C2
20[bridge]
21B1B1B1B1B1B1
22B1B1B1B1B1B1
23[bridge 2]
24B2B2B2B2B2B2
25B2B2B2B2B2B2
26[prechorus]
27PRE1PRE1PRE1
28PRE1PRE1PRE1
29[prechorus 2]
30PRE2PRE2PRE2
31PRE2PRE2PRE2
32[ending]
33ENDENDENDEND
34ENDENDENDEND</Contents>
35 <Notations />
36 <Sequence>1,2,c,t,b,w,p,q,e</Sequence>
37 <Writer />
38 <Copyright />
39 <Category />
40 <Timing />
41 <MusicKey />
42 <Capo>-1</Capo>
43 <LicenceAdmin1 />
44 <LicenceAdmin2 />
45 <BookReference />
46 <UserReference />
47 <FormatData />
48 <Settings>10=&gt;</Settings>
49 </Item>
50</EasiSlides>
0\ No newline at end of file51\ No newline at end of file
152
=== modified file 'tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json'
--- tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json 2015-12-17 21:39:52 +0000
+++ tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json 2017-01-20 20:51:35 +0000
@@ -1,4 +1,4 @@
1{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Copyright:"Public1{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Capo:-1,Copyright:"Public
2Domain",Theme:"tema12Domain",Theme:"tema1
3tema2",CCLI:"12345",Alias:"A safe stronghold",Memo1:"This is3tema2",CCLI:"12345",Alias:"A safe stronghold",Memo1:"This is
4the first comment4the first comment