Merge lp:~tomasgroth/openlp/song-import-fixes25 into lp:openlp

[Tomas Groth]

Clean search lyrics for formatting tags. Fixes bug #1655988.
Fix an issue with easyslide import not handling verse order correctly. Fixes bug #1655985.
Improve the songbeamer encoding detection. Fixes bug #1530597.
Handle a few videopsalm quirks. Fixes bug #1652851.

[Tomas Groth (tomasgroth)]

lp:~tomasgroth/openlp/song-import-fixes25 (revision 2718)
[SUCCESS] https://ci.openlp.io/job/Branch-01-Pull/1892/
[SUCCESS] https://ci.openlp.io/job/Branch-02-Functional-Tests/1803/
[SUCCESS] https://ci.openlp.io/job/Branch-03-Interface-Tests/1742/
[SUCCESS] https://ci.openlp.io/job/Branch-04a-Windows_Functional_Tests/1478/
[SUCCESS] https://ci.openlp.io/job/Branch-04b-Windows_Interface_Tests/1068/
[SUCCESS] https://ci.openlp.io/job/Branch-05a-Code_Analysis/1136/
[SUCCESS] https://ci.openlp.io/job/Branch-05b-Test_Coverage/1004/
[SUCCESS] https://ci.openlp.io/job/Branch-05c-Code_Analysis2/148/

[Tim Bentley (trb143)]

review: Approve

=== modified file 'openlp/plugins/songs/lib/__init__.py'
--- openlp/plugins/songs/lib/__init__.py	2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/__init__.py	2017-01-20 20:51:35 +0000
@@ -30,7 +30,7 @@
 from PyQt5 import QtWidgets
 
 from openlp.core.common import AppLocation, CONTROL_CHARS
-from openlp.core.lib import translate
+from openlp.core.lib import translate, clean_tags
 from openlp.plugins.songs.lib.db import Author, MediaFile, Song, Topic
 from openlp.plugins.songs.lib.ui import SongStrings
 
@@ -380,7 +380,7 @@
     if isinstance(song.lyrics, bytes):
         song.lyrics = str(song.lyrics, encoding='utf8')
     verses = SongXML().get_verses(song.lyrics)
-    song.search_lyrics = ' '.join([clean_string(verse[1]) for verse in verses])
+    song.search_lyrics = ' '.join([clean_string(clean_tags(verse[1])) for verse in verses])
     # The song does not have any author, add one.
     if not song.authors_songs:
         name = SongStrings.AuthorUnknown

=== modified file 'openlp/plugins/songs/lib/importers/easyslides.py'
--- openlp/plugins/songs/lib/importers/easyslides.py	2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/easyslides.py	2017-01-20 20:51:35 +0000
@@ -180,7 +180,7 @@
         reg = default_region
         verses[reg] = {}
         # instance differentiates occurrences of same verse tag
-        vt = 'V'
+        vt = 'v'
         vn = '1'
         inst = 1
         for line in lines:
@@ -193,14 +193,14 @@
                         inst += 1
                 else:
                     # separators are not used, so empty line starts a new verse
-                    vt = 'V'
+                    vt = 'v'
                     vn = len(verses[reg].get(vt, {})) + 1
                     inst = 1
             elif line[0:7] == '[region':
                 reg = self._extract_region(line)
                 verses.setdefault(reg, {})
                 if not regions_in_verses:
-                    vt = 'V'
+                    vt = 'v'
                     vn = '1'
                     inst = 1
             elif line[0] == '[':
@@ -213,7 +213,7 @@
                 if match:
                     marker = match.group(1).strip()
                     vn = match.group(2)
-                vt = MarkTypes.get(marker, 'O') if marker else 'V'
+                vt = MarkTypes.get(marker, 'o') if marker else 'v'
                 if regions_in_verses:
                     region = default_region
                 inst = 1
@@ -238,13 +238,13 @@
                 lines = '\n'.join(verses[reg][vt][vn][inst])
                 self.add_verse(lines, versetag)
         SeqTypes = {
-            'p': 'P1',
-            'q': 'P2',
-            'c': 'C1',
-            't': 'C2',
-            'b': 'B1',
-            'w': 'B2',
-            'e': 'E1'}
+            'p': 'p1',
+            'q': 'p2',
+            'c': 'c1',
+            't': 'c2',
+            'b': 'b1',
+            'w': 'b2',
+            'e': 'e1'}
         # Make use of Sequence data, determining the order of verses
         try:
             order = str(song.Sequence).strip().split(',')
@@ -252,7 +252,7 @@
                 if not tag:
                     continue
                 elif tag[0].isdigit():
-                    tag = 'V' + tag
+                    tag = 'v' + tag
                 elif tag.lower() in SeqTypes:
                     tag = SeqTypes[tag.lower()]
                 else:

=== modified file 'openlp/plugins/songs/lib/importers/songbeamer.py'
--- openlp/plugins/songs/lib/importers/songbeamer.py	2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/songbeamer.py	2017-01-20 20:51:35 +0000
@@ -28,6 +28,7 @@
 import os
 import re
 
+from openlp.core.common import get_file_encoding
 from openlp.plugins.songs.lib import VerseType
 from openlp.plugins.songs.lib.importers.songimport import SongImport
 
@@ -113,13 +114,15 @@
             read_verses = False
             file_name = os.path.split(import_file)[1]
             if os.path.isfile(import_file):
-                # First open in binary mode to detect the encoding
-                detect_file = open(import_file, 'rb')
-                details = chardet.detect(detect_file.read())
-                detect_file.close()
-                infile = codecs.open(import_file, 'r', details['encoding'])
+                # Detect the encoding
+                self.input_file_encoding = get_file_encoding(import_file)['encoding']
+                # The encoding should only be ANSI (cp1252), UTF-8, Unicode, Big-Endian-Unicode.
+                # So if it doesn't start with 'u' we default to cp1252. See:
+                # https://forum.songbeamer.com/viewtopic.php?p=419&sid=ca4814924e37c11e4438b7272a98b6f2
+                if self.input_file_encoding.lower().startswith('u'):
+                    self.input_file_encoding = 'cp1252'
+                infile = open(import_file, 'rt', encoding=self.input_file_encoding)
                 song_data = infile.readlines()
-                infile.close()
             else:
                 continue
             self.title = file_name.split('.sng')[0]

=== modified file 'openlp/plugins/songs/lib/importers/videopsalm.py'
--- openlp/plugins/songs/lib/importers/videopsalm.py	2016-12-31 11:01:36 +0000
+++ openlp/plugins/songs/lib/importers/videopsalm.py	2017-01-20 20:51:35 +0000
@@ -65,8 +65,8 @@
                 if c == '\n':
                     if inside_quotes:
                         processed_content += '\\n'
-                # Put keys in quotes
-                elif c.isalnum() and not inside_quotes:
+                # Put keys in quotes. The '-' is for handling nagative numbers
+                elif (c.isalnum() or c == '-') and not inside_quotes:
                     processed_content += '"' + c
                     c = next(file_content_it)
                     while c.isalnum():
@@ -121,6 +121,8 @@
                 if 'Memo3' in song:
                     self.add_comment(song['Memo3'])
                 for verse in song['Verses']:
+                    if 'Text' not in verse:
+                        continue
                     self.add_verse(verse['Text'], 'v')
                 if not self.finish():
                     self.log_error('Could not import {title}'.format(title=self.title))

=== modified file 'tests/functional/openlp_plugins/songs/test_easyslidesimport.py'
--- tests/functional/openlp_plugins/songs/test_easyslidesimport.py	2016-12-31 11:01:36 +0000
+++ tests/functional/openlp_plugins/songs/test_easyslidesimport.py	2017-01-20 20:51:35 +0000
@@ -43,3 +43,5 @@
         """
         self.file_import(os.path.join(TEST_PATH, 'amazing-grace.xml'),
                          self.load_external_result_data(os.path.join(TEST_PATH, 'Amazing Grace.json')))
+        self.file_import(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.xml'),
+                         self.load_external_result_data(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.json')))

=== modified file 'tests/resources/easyslidessongs/Amazing Grace.json'
--- tests/resources/easyslidessongs/Amazing Grace.json	2016-01-08 21:42:36 +0000
+++ tests/resources/easyslidessongs/Amazing Grace.json	2017-01-20 20:51:35 +0000
@@ -6,27 +6,27 @@
     "verses": [
         [
             "Amazing grace! How sweet the sound\nThat saved a wretch like me;\nI once was lost, but now am found,\nWas blind, but now I see.",
-            "V1"
+            "v1"
         ],
         [
             "'Twas grace that taught my heart to fear,\nAnd grace my fears relieved;\nHow precious did that grace appear,\nThe hour I first believed!",
-            "V2"
+            "v2"
         ],
         [
             "Through many dangers, toils and snares\nI have already come;\n'Tis grace that brought me safe thus far,\nAnd grace will lead me home.",
-            "V3"
+            "v3"
         ],
         [
             "The Lord has promised good to me,\nHis word my hope secures;\nHe will my shield and portion be\nAs long as life endures.",
-            "V4"
+            "v4"
         ],
         [
             "Yes, when this heart and flesh shall fail,\nAnd mortal life shall cease,\nI shall possess within the veil\nA life of joy and peace.",
-            "V5"
+            "v5"
         ],
         [
             "When we've been there a thousand years,\nBright shining as the sun,\nWe've no less days to sing God's praise\nThan when we first begun.",
-            "V6"
+            "v6"
         ]
     ]
 }

=== added file 'tests/resources/easyslidessongs/Export_2017-01-12_BB.json'
--- tests/resources/easyslidessongs/Export_2017-01-12_BB.json	1970-01-01 00:00:00 +0000
+++ tests/resources/easyslidessongs/Export_2017-01-12_BB.json	2017-01-20 20:51:35 +0000
@@ -0,0 +1,44 @@
+{
+    "title": "BBBBBBBBB",
+    "authors": [
+        "John Newton (1725-1807)"
+    ],
+    "verses": [
+        [
+            "V1V1V1V1V1V1\nV1V1V1V1V1V1",
+            "v1"
+        ],
+        [
+            "V2V2V2V2V2V2\nV2V2V2V2V2V2",
+            "v2"
+        ],
+        [
+            "C1C1C1C1C1C1\nC1C1C1C1C1C1",
+            "c1"
+        ],
+        [
+            "C2C2C2C2C2C2\nC2C2C2C2C2C2",
+            "c2"
+        ],
+        [
+            "B1B1B1B1B1B1\nB1B1B1B1B1B1",
+            "b1"
+        ],
+        [
+            "B2B2B2B2B2B2\nB2B2B2B2B2B2",
+            "b2"
+        ],
+        [
+            "PRE1PRE1PRE1\nPRE1PRE1PRE1",
+            "p1"
+        ],
+        [
+            "PRE2PRE2PRE2\nPRE2PRE2PRE2",
+            "p2"
+        ],
+        [
+            "ENDENDENDEND\nENDENDENDEND",
+            "e1"
+        ]
+    ]
+}

=== added file 'tests/resources/easyslidessongs/Export_2017-01-12_BB.xml'
--- tests/resources/easyslidessongs/Export_2017-01-12_BB.xml	1970-01-01 00:00:00 +0000
+++ tests/resources/easyslidessongs/Export_2017-01-12_BB.xml	2017-01-20 20:51:35 +0000
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EasiSlides>
+  <Item>
+    <Title1>BBBBBBBBB</Title1>
+    <Title2 />
+    <Folder>NAGY</Folder>
+    <SongNumber>0</SongNumber>
+    <Contents>[1]
+V1V1V1V1V1V1
+V1V1V1V1V1V1
+[2]
+V2V2V2V2V2V2
+V2V2V2V2V2V2
+[chorus]
+C1C1C1C1C1C1
+C1C1C1C1C1C1
+[chorus 2]
+C2C2C2C2C2C2
+C2C2C2C2C2C2
+[bridge]
+B1B1B1B1B1B1
+B1B1B1B1B1B1
+[bridge 2]
+B2B2B2B2B2B2
+B2B2B2B2B2B2
+[prechorus]
+PRE1PRE1PRE1
+PRE1PRE1PRE1
+[prechorus 2]
+PRE2PRE2PRE2
+PRE2PRE2PRE2
+[ending]
+ENDENDENDEND
+ENDENDENDEND</Contents>
+    <Notations />
+    <Sequence>1,2,c,t,b,w,p,q,e</Sequence>
+    <Writer />
+    <Copyright />
+    <Category />
+    <Timing />
+    <MusicKey />
+    <Capo>-1</Capo>
+    <LicenceAdmin1 />
+    <LicenceAdmin2 />
+    <BookReference />
+    <UserReference />
+    <FormatData />
+    <Settings>10=&gt;</Settings>
+  </Item>
+</EasiSlides>
\ No newline at end of file

=== modified file 'tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json'
--- tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json	2015-12-17 21:39:52 +0000
+++ tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json	2017-01-20 20:51:35 +0000
@@ -1,4 +1,4 @@
-{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Copyright:"Public
+{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Capo:-1,Copyright:"Public
 Domain",Theme:"tema1
 tema2",CCLI:"12345",Alias:"A safe stronghold",Memo1:"This is
 the first comment