Merge lp:~tomasgroth/openlp/song-import-fixes24 into lp:openlp/2.4

[Tomas Groth]

Fixes:
Bug #1487788: Importing photos does not give focus to OpenLP
Bug #1512040: Loop tooltip gets stuck to "Stop playing..."
Bug #1624661: Missing DB in unmounted disk results in Traceback
Clean search lyrics for formatting tags. Fixes bug #1655988.
Fix an issue with easyslide import not handling verse order correctly. Fixes bug #1655985.
Improve the songbeamer encoding detection. Fixes bug #1530597.
Handle a few videopsalm quirks. Fixes bug #1652851.

[Tomas Groth (tomasgroth)]

lp:~tomasgroth/openlp/song-import-fixes24 (revision 2667)
[SUCCESS] https://ci.openlp.io/job/Branch-01-Pull/1893/
[SUCCESS] https://ci.openlp.io/job/Branch-02-Functional-Tests/1804/
[SUCCESS] https://ci.openlp.io/job/Branch-03-Interface-Tests/1743/
[SUCCESS] https://ci.openlp.io/job/Branch-04a-Windows_Functional_Tests/1479/
[SUCCESS] https://ci.openlp.io/job/Branch-04b-Windows_Interface_Tests/1069/
[SUCCESS] https://ci.openlp.io/job/Branch-05a-Code_Analysis/1137/
[SUCCESS] https://ci.openlp.io/job/Branch-05b-Test_Coverage/1005/

[Tim Bentley (trb143)]

review: Approve

=== modified file 'openlp/core/__init__.py'
--- openlp/core/__init__.py	2016-12-31 11:05:48 +0000
+++ openlp/core/__init__.py	2017-01-22 20:24:55 +0000
@@ -177,6 +177,38 @@
             self.shared_memory.create(1)
             return False
 
+    def is_data_path_missing(self):
+        """
+        Check if the data folder path exists.
+        """
+        data_folder_path = AppLocation.get_data_path()
+        if not os.path.exists(data_folder_path):
+            log.critical('Database was not found in: ' + data_folder_path)
+            status = QtWidgets.QMessageBox.critical(None, translate('OpenLP', 'Data Directory Error'),
+                                                    translate('OpenLP', 'OpenLP data folder was not found in:\n\n{path}'
+                                                                        '\n\nThe location of the data folder was '
+                                                                        'previously changed from the OpenLP\'s '
+                                                                        'default location. If the data was stored on '
+                                                                        'removable device, that device needs to be '
+                                                                        'made available.\n\nYou may reset the data '
+                                                                        'location back to the default location, '
+                                                                        'or you can try to make the current location '
+                                                                        'available.\n\nDo you want to reset to the '
+                                                                        'default data location? If not, OpenLP will be '
+                                                                        'closed so you can try to fix the the problem.')
+                                                    .format(path=data_folder_path),
+                                                    QtWidgets.QMessageBox.StandardButtons(QtWidgets.QMessageBox.Yes |
+                                                                                          QtWidgets.QMessageBox.No),
+                                                    QtWidgets.QMessageBox.No)
+            if status == QtWidgets.QMessageBox.No:
+                # If answer was "No", return "True", it will shutdown OpenLP in def main
+                log.info('User requested termination')
+                return True
+            # If answer was "Yes", remove the custom data path thus resetting the default location.
+            Settings().remove('advanced/data path')
+            log.info('Database location has been reset to the default settings.')
+            return False
+
     def hook_exception(self, exc_type, value, traceback):
         """
         Add an exception hook so that any uncaught exceptions are displayed in this window rather than somewhere where
@@ -213,7 +245,7 @@
             Settings().setValue('core/application version', openlp_version)
         # If data_version is different from the current version ask if we should backup the data folder
         elif data_version != openlp_version:
-            if self.splash.isVisible():
+            if can_show_splash and self.splash.isVisible():
                 self.splash.hide()
             if QtWidgets.QMessageBox.question(None, translate('OpenLP', 'Backup'),
                                               translate('OpenLP', 'OpenLP has been upgraded, do you want to create '
@@ -378,9 +410,13 @@
     Registry.create()
     Registry().register('application', application)
     application.setApplicationVersion(get_application_version()['version'])
-    # Instance check
+    # Check if an instance of OpenLP is already running. Quit if there is a running instance and the user only wants one
     if application.is_already_running():
         sys.exit()
+    # If the custom data path is missing and the user wants to restore the data path, quit OpenLP.
+    if application.is_data_path_missing():
+        application.shared_memory.detach()
+        sys.exit()
     # Remove/convert obsolete settings.
     Settings().remove_obsolete_settings()
     # First time checks in settings

=== modified file 'openlp/core/lib/treewidgetwithdnd.py'
--- openlp/core/lib/treewidgetwithdnd.py	2016-12-31 11:05:48 +0000
+++ openlp/core/lib/treewidgetwithdnd.py	2017-01-22 20:24:55 +0000
@@ -26,7 +26,7 @@
 
 from PyQt5 import QtCore, QtGui, QtWidgets
 
-from openlp.core.common import Registry
+from openlp.core.common import Registry, is_win
 
 
 class TreeWidgetWithDnD(QtWidgets.QTreeWidget):
@@ -108,6 +108,11 @@
 
         :param event: Handle of the event pint passed
         """
+        # If we are on Windows, OpenLP window will not be set on top. For example, user can drag images to Library and
+        # the folder stays on top of the group creation box. This piece of code fixes this issue.
+        if is_win():
+            self.setWindowState(self.windowState() & ~QtCore.Qt.WindowMinimized | QtCore.Qt.WindowActive)
+            self.setWindowState(QtCore.Qt.WindowNoState)
         if event.mimeData().hasUrls():
             event.setDropAction(QtCore.Qt.CopyAction)
             event.accept()

=== modified file 'openlp/core/ui/slidecontroller.py'
--- openlp/core/ui/slidecontroller.py	2016-12-31 11:05:48 +0000
+++ openlp/core/ui/slidecontroller.py	2017-01-22 20:24:55 +0000
@@ -714,8 +714,10 @@
         # Reset the button
         self.play_slides_once.setChecked(False)
         self.play_slides_once.setIcon(build_icon(':/media/media_time.png'))
+        self.play_slides_once.setText(UiStrings().PlaySlidesToEnd)
         self.play_slides_loop.setChecked(False)
         self.play_slides_loop.setIcon(build_icon(':/media/media_time.png'))
+        self.play_slides_loop.setText(UiStrings().PlaySlidesInLoop)
         if item.is_text():
             if (Settings().value(self.main_window.songs_settings_section + '/display songbar') and
                     not self.song_menu.menu().isEmpty()):

=== modified file 'openlp/plugins/songs/lib/__init__.py'
--- openlp/plugins/songs/lib/__init__.py	2016-12-31 11:05:48 +0000
+++ openlp/plugins/songs/lib/__init__.py	2017-01-22 20:24:55 +0000
@@ -30,7 +30,7 @@
 from PyQt5 import QtWidgets
 
 from openlp.core.common import AppLocation
-from openlp.core.lib import translate
+from openlp.core.lib import translate, clean_tags
 from openlp.core.utils import CONTROL_CHARS
 from openlp.plugins.songs.lib.db import Author, MediaFile, Song, Topic
 from openlp.plugins.songs.lib.ui import SongStrings
@@ -381,7 +381,7 @@
     if isinstance(song.lyrics, bytes):
         song.lyrics = str(song.lyrics, encoding='utf8')
     verses = SongXML().get_verses(song.lyrics)
-    song.search_lyrics = ' '.join([clean_string(verse[1]) for verse in verses])
+    song.search_lyrics = ' '.join([clean_string(clean_tags(verse[1])) for verse in verses])
     # The song does not have any author, add one.
     if not song.authors_songs:
         name = SongStrings.AuthorUnknown

=== modified file 'openlp/plugins/songs/lib/importers/easyslides.py'
--- openlp/plugins/songs/lib/importers/easyslides.py	2016-12-31 11:05:48 +0000
+++ openlp/plugins/songs/lib/importers/easyslides.py	2017-01-22 20:24:55 +0000
@@ -179,7 +179,7 @@
         reg = default_region
         verses[reg] = {}
         # instance differentiates occurrences of same verse tag
-        vt = 'V'
+        vt = 'v'
         vn = '1'
         inst = 1
         for line in lines:
@@ -192,14 +192,14 @@
                         inst += 1
                 else:
                     # separators are not used, so empty line starts a new verse
-                    vt = 'V'
+                    vt = 'v'
                     vn = len(verses[reg].get(vt, {})) + 1
                     inst = 1
             elif line[0:7] == '[region':
                 reg = self._extract_region(line)
                 verses.setdefault(reg, {})
                 if not regions_in_verses:
-                    vt = 'V'
+                    vt = 'v'
                     vn = '1'
                     inst = 1
             elif line[0] == '[':
@@ -212,7 +212,7 @@
                 if match:
                     marker = match.group(1).strip()
                     vn = match.group(2)
-                vt = MarkTypes.get(marker, 'O') if marker else 'V'
+                vt = MarkTypes.get(marker, 'o') if marker else 'v'
                 if regions_in_verses:
                     region = default_region
                 inst = 1
@@ -237,13 +237,13 @@
                 lines = '\n'.join(verses[reg][vt][vn][inst])
                 self.add_verse(lines, versetag)
         SeqTypes = {
-            'p': 'P1',
-            'q': 'P2',
-            'c': 'C1',
-            't': 'C2',
-            'b': 'B1',
-            'w': 'B2',
-            'e': 'E1'}
+            'p': 'p1',
+            'q': 'p2',
+            'c': 'c1',
+            't': 'c2',
+            'b': 'b1',
+            'w': 'b2',
+            'e': 'e1'}
         # Make use of Sequence data, determining the order of verses
         try:
             order = str(song.Sequence).strip().split(',')
@@ -251,7 +251,7 @@
                 if not tag:
                     continue
                 elif tag[0].isdigit():
-                    tag = 'V' + tag
+                    tag = 'v' + tag
                 elif tag.lower() in SeqTypes:
                     tag = SeqTypes[tag.lower()]
                 else:

=== modified file 'openlp/plugins/songs/lib/importers/songbeamer.py'
--- openlp/plugins/songs/lib/importers/songbeamer.py	2016-12-31 11:05:48 +0000
+++ openlp/plugins/songs/lib/importers/songbeamer.py	2017-01-22 20:24:55 +0000
@@ -115,11 +115,15 @@
             if os.path.isfile(import_file):
                 # First open in binary mode to detect the encoding
                 detect_file = open(import_file, 'rb')
-                details = chardet.detect(detect_file.read())
+                self.input_file_encoding = chardet.detect(detect_file.read())['encoding']
                 detect_file.close()
-                infile = codecs.open(import_file, 'r', details['encoding'])
+                # The encoding should only be ANSI (cp1252), UTF-8, Unicode, Big-Endian-Unicode.
+                # So if it doesn't start with 'u' we default to cp1252. See:
+                # https://forum.songbeamer.com/viewtopic.php?p=419&sid=ca4814924e37c11e4438b7272a98b6f2
+                if self.input_file_encoding.lower().startswith('u'):
+                    self.input_file_encoding = 'cp1252'
+                infile = open(import_file, 'rt', encoding=self.input_file_encoding)
                 song_data = infile.readlines()
-                infile.close()
             else:
                 continue
             self.title = file_name.split('.sng')[0]

=== modified file 'openlp/plugins/songs/lib/importers/videopsalm.py'
--- openlp/plugins/songs/lib/importers/videopsalm.py	2016-12-31 11:05:48 +0000
+++ openlp/plugins/songs/lib/importers/videopsalm.py	2017-01-22 20:24:55 +0000
@@ -65,8 +65,8 @@
                 if c == '\n':
                     if inside_quotes:
                         processed_content += '\\n'
-                # Put keys in quotes
-                elif c.isalnum() and not inside_quotes:
+                # Put keys in quotes. The '-' is for handling nagative numbers
+                elif (c.isalnum() or c == '-') and not inside_quotes:
                     processed_content += '"' + c
                     c = next(file_content_it)
                     while c.isalnum():
@@ -121,6 +121,8 @@
                 if 'Memo3' in song:
                     self.add_comment(song['Memo3'])
                 for verse in song['Verses']:
+                    if 'Text' not in verse:
+                        continue
                     self.add_verse(verse['Text'], 'v')
                 if not self.finish():
                     self.log_error('Could not import %s' % self.title)

=== modified file 'tests/functional/openlp_plugins/songs/test_easyslidesimport.py'
--- tests/functional/openlp_plugins/songs/test_easyslidesimport.py	2016-12-31 11:05:48 +0000
+++ tests/functional/openlp_plugins/songs/test_easyslidesimport.py	2017-01-22 20:24:55 +0000
@@ -43,3 +43,5 @@
         """
         self.file_import(os.path.join(TEST_PATH, 'amazing-grace.xml'),
                          self.load_external_result_data(os.path.join(TEST_PATH, 'Amazing Grace.json')))
+        self.file_import(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.xml'),
+                         self.load_external_result_data(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.json')))

=== modified file 'tests/resources/easyslidessongs/Amazing Grace.json'
--- tests/resources/easyslidessongs/Amazing Grace.json	2016-01-08 21:42:36 +0000
+++ tests/resources/easyslidessongs/Amazing Grace.json	2017-01-22 20:24:55 +0000
@@ -6,27 +6,27 @@
     "verses": [
         [
             "Amazing grace! How sweet the sound\nThat saved a wretch like me;\nI once was lost, but now am found,\nWas blind, but now I see.",
-            "V1"
+            "v1"
         ],
         [
             "'Twas grace that taught my heart to fear,\nAnd grace my fears relieved;\nHow precious did that grace appear,\nThe hour I first believed!",
-            "V2"
+            "v2"
         ],
         [
             "Through many dangers, toils and snares\nI have already come;\n'Tis grace that brought me safe thus far,\nAnd grace will lead me home.",
-            "V3"
+            "v3"
         ],
         [
             "The Lord has promised good to me,\nHis word my hope secures;\nHe will my shield and portion be\nAs long as life endures.",
-            "V4"
+            "v4"
         ],
         [
             "Yes, when this heart and flesh shall fail,\nAnd mortal life shall cease,\nI shall possess within the veil\nA life of joy and peace.",
-            "V5"
+            "v5"
         ],
         [
             "When we've been there a thousand years,\nBright shining as the sun,\nWe've no less days to sing God's praise\nThan when we first begun.",
-            "V6"
+            "v6"
         ]
     ]
 }

=== added file 'tests/resources/easyslidessongs/Export_2017-01-12_BB.json'
--- tests/resources/easyslidessongs/Export_2017-01-12_BB.json	1970-01-01 00:00:00 +0000
+++ tests/resources/easyslidessongs/Export_2017-01-12_BB.json	2017-01-22 20:24:55 +0000
@@ -0,0 +1,44 @@
+{
+    "title": "BBBBBBBBB",
+    "authors": [
+        "John Newton (1725-1807)"
+    ],
+    "verses": [
+        [
+            "V1V1V1V1V1V1\nV1V1V1V1V1V1",
+            "v1"
+        ],
+        [
+            "V2V2V2V2V2V2\nV2V2V2V2V2V2",
+            "v2"
+        ],
+        [
+            "C1C1C1C1C1C1\nC1C1C1C1C1C1",
+            "c1"
+        ],
+        [
+            "C2C2C2C2C2C2\nC2C2C2C2C2C2",
+            "c2"
+        ],
+        [
+            "B1B1B1B1B1B1\nB1B1B1B1B1B1",
+            "b1"
+        ],
+        [
+            "B2B2B2B2B2B2\nB2B2B2B2B2B2",
+            "b2"
+        ],
+        [
+            "PRE1PRE1PRE1\nPRE1PRE1PRE1",
+            "p1"
+        ],
+        [
+            "PRE2PRE2PRE2\nPRE2PRE2PRE2",
+            "p2"
+        ],
+        [
+            "ENDENDENDEND\nENDENDENDEND",
+            "e1"
+        ]
+    ]
+}

=== added file 'tests/resources/easyslidessongs/Export_2017-01-12_BB.xml'
--- tests/resources/easyslidessongs/Export_2017-01-12_BB.xml	1970-01-01 00:00:00 +0000
+++ tests/resources/easyslidessongs/Export_2017-01-12_BB.xml	2017-01-22 20:24:55 +0000
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EasiSlides>
+  <Item>
+    <Title1>BBBBBBBBB</Title1>
+    <Title2 />
+    <Folder>NAGY</Folder>
+    <SongNumber>0</SongNumber>
+    <Contents>[1]
+V1V1V1V1V1V1
+V1V1V1V1V1V1
+[2]
+V2V2V2V2V2V2
+V2V2V2V2V2V2
+[chorus]
+C1C1C1C1C1C1
+C1C1C1C1C1C1
+[chorus 2]
+C2C2C2C2C2C2
+C2C2C2C2C2C2
+[bridge]
+B1B1B1B1B1B1
+B1B1B1B1B1B1
+[bridge 2]
+B2B2B2B2B2B2
+B2B2B2B2B2B2
+[prechorus]
+PRE1PRE1PRE1
+PRE1PRE1PRE1
+[prechorus 2]
+PRE2PRE2PRE2
+PRE2PRE2PRE2
+[ending]
+ENDENDENDEND
+ENDENDENDEND</Contents>
+    <Notations />
+    <Sequence>1,2,c,t,b,w,p,q,e</Sequence>
+    <Writer />
+    <Copyright />
+    <Category />
+    <Timing />
+    <MusicKey />
+    <Capo>-1</Capo>
+    <LicenceAdmin1 />
+    <LicenceAdmin2 />
+    <BookReference />
+    <UserReference />
+    <FormatData />
+    <Settings>10=&gt;</Settings>
+  </Item>
+</EasiSlides>
\ No newline at end of file

=== modified file 'tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json'
--- tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json	2015-12-17 21:39:52 +0000
+++ tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json	2017-01-22 20:24:55 +0000
@@ -1,4 +1,4 @@
-{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Copyright:"Public
+{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Capo:-1,Copyright:"Public
 Domain",Theme:"tema1
 tema2",CCLI:"12345",Alias:"A safe stronghold",Memo1:"This is
 the first comment