Merge lp:~john-peterson/calibre/meta into lp:calibre

[John Peterson]

[John Peterson (john-peterson)]

Comments exist at https://github.com/mirror/calibre/pull/3

[Kovid Goyal (kovid)]

Since you still insist on using github for comments and from your general tone, I conclude that dealing with you is more trouble than it is worth. I will not be accepting any patches from you in the future, so dont bother submitting any.

review: Disapprove

[John Peterson (john-peterson)]

A reply is posted at https://github.com/mirror/calibre/pull/3

Unmerged revisions

14248. By John Peterson on 2013-02-01

Adding option to auto add metadata on import

Coordinating add actions so that convert waits for metadata adding

Narrowing the metadata download modification check to avoid checking for changes in custom metadata (which is not downloaded or overwritten)

Adding option to clear IDs before metadata download (to obtain the default edition)

Adding option to not confirm before updating metadata in library

14247. By John Peterson on 2013-02-01

Adding postimport plugin call to manual add

=== modified file 'src/calibre/ebooks/metadata/sources/base.py'
--- src/calibre/ebooks/metadata/sources/base.py	2013-01-31 15:49:28 +0000
+++ src/calibre/ebooks/metadata/sources/base.py	2013-02-01 04:56:24 +0000
@@ -18,6 +18,10 @@
 from calibre.ebooks.metadata import check_isbn
 
 msprefs = JSONConfig('metadata_sources/global.json')
+msprefs.defaults['confirm'] = True
+msprefs.defaults['auto_add'] = False
+msprefs.defaults['auto_overwrite'] = False
+msprefs.defaults['auto_clear'] = False
 msprefs.defaults['txt_comments'] = False
 msprefs.defaults['ignore_fields'] = []
 msprefs.defaults['user_default_ignore_fields'] = []

=== modified file 'src/calibre/gui2/actions/edit_metadata.py'
--- src/calibre/gui2/actions/edit_metadata.py	2012-10-05 06:00:02 +0000
+++ src/calibre/gui2/actions/edit_metadata.py	2013-02-01 04:56:24 +0000
@@ -67,18 +67,18 @@
         self.action_merge.setEnabled(enabled)
 
     # Download metadata {{{
-    def download_metadata(self, ids=None, ensure_fields=None):
+    def download_metadata(self, ids=None, ensure_fields=None, confirm=True, clear=False, identify=True, covers=True, callback_done=None):
+        db = self.gui.library_view.model().db
         if ids is None:
             rows = self.gui.library_view.selectionModel().selectedRows()
             if not rows or len(rows) == 0:
                 return error_dialog(self.gui, _('Cannot download metadata'),
                             _('No books selected'), show=True)
-            db = self.gui.library_view.model().db
             ids = [db.id(row.row()) for row in rows]
         from calibre.gui2.metadata.bulk_download import start_download
         start_download(self.gui, ids,
                 Dispatcher(self.metadata_downloaded),
-                ensure_fields=ensure_fields)
+                ensure_fields=ensure_fields, confirm=confirm, clear=clear, identify=identify, covers=covers, callback_done=callback_done)
 
     def cleanup_bulk_download(self, tdir, *args):
         try:
@@ -87,6 +87,11 @@
             pass
 
     def metadata_downloaded(self, job):
+        self._metadata_downloaded(job)
+        if job.callback_done:
+            job.callback_done()
+
+    def _metadata_downloaded(self, job):
         if job.failed:
             self.gui.job_exception(job, dialog_title=_('Failed to download metadata'))
             return
@@ -118,16 +123,21 @@
                     'after updating metadata')
 
         payload = (id_map, tdir, log_file, lm_map,
-                failed_ids.union(failed_covers))
-        self.gui.proceed_question(self.apply_downloaded_metadata, payload,
-                log_file, _('Download log'), _('Download complete'), msg,
-                det_msg=det_msg, show_copy_button=show_copy_button,
-                cancel_callback=partial(self.cleanup_bulk_download, tdir),
-                log_is_file=True, checkbox_msg=checkbox_msg,
-                checkbox_checked=False)
+                failed_ids.union(failed_covers), job.old_metadata)
+              
+        from calibre.ebooks.metadata.sources.base import msprefs
+        if msprefs['confirm']:
+            self.gui.proceed_question(self.apply_downloaded_metadata, payload,
+                    log_file, _('Download log'), _('Download complete'), msg,
+                    det_msg=det_msg, show_copy_button=show_copy_button,
+                    cancel_callback=partial(self.cleanup_bulk_download, tdir),
+                    log_is_file=True, checkbox_msg=checkbox_msg,
+                    checkbox_checked=False)
+        else:
+            self.apply_downloaded_metadata(payload)
 
     def apply_downloaded_metadata(self, payload, *args):
-        good_ids, tdir, log_file, lm_map, failed_ids = payload
+        good_ids, tdir, log_file, lm_map, failed_ids, old_metadata = payload
         if not good_ids:
             return
 
@@ -135,8 +145,7 @@
         db = self.gui.current_db
 
         for i in good_ids:
-            lm = db.metadata_last_modified(i, index_is_id=True)
-            if lm is not None and lm_map[i] is not None and lm > lm_map[i]:
+            if db.check_if_standard_metadata_modified(i, index_is_id=True, other=old_metadata[i]):
                 title = db.title(i, index_is_id=True)
                 authors = db.authors(i, index_is_id=True)
                 if authors:

=== modified file 'src/calibre/gui2/add.py'
--- src/calibre/gui2/add.py	2013-01-06 08:22:38 +0000
+++ src/calibre/gui2/add.py	2013-02-01 04:56:24 +0000
@@ -9,7 +9,7 @@
 
 from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.gui2 import (error_dialog, info_dialog, gprefs,
-        warning_dialog, available_width)
+        warning_dialog, available_width, Dispatcher)
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.metadata import MetaInformation
 from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG
@@ -111,15 +111,49 @@
         self.auto_convert_books = set()
 
     def end(self):
+        from calibre.ebooks.metadata.sources.base import msprefs        
+        callback_done = None
+        ids_add_metadata = set()
+    
+        # check if metadata should be added
+        if msprefs['auto_overwrite'] or msprefs['auto_clear']:
+            ids_add_metadata = self.auto_convert_books
+            dl_identify = True
+            dl_covers = True
+        
+        if msprefs['auto_add'] and not (msprefs['auto_overwrite'] or msprefs['auto_clear']):
+            dl_identify = False
+            dl_covers = False
+            for id in self.auto_convert_books:
+                mi = self.db.get_metadata(id, index_is_id=True, get_cover=True)
+                missing_identify = bool(not (mi and mi.isbn and mi.pubdate and mi.comments))
+                missing_cover = bool(not (mi and mi.cover))
+                if missing_identify: dl_identify = True
+                if missing_cover: dl_covers = True
+                if missing_identify or missing_cover:
+                    ids_add_metadata.add(id)
+        
+        # auto convert now or after metadata is added
         if (gprefs['manual_add_auto_convert'] and
                 self.auto_convert_books):
+            if ids_add_metadata:
+                callback_done = Dispatcher(self.auto_convert)
+            else:
+                self.auto_convert()
+
+        if ids_add_metadata:
             from calibre.gui2.ui import get_gui
             gui = get_gui()
-            gui.iactions['Convert Books'].auto_convert_auto_add(
-                self.auto_convert_books)
+            gui.iactions['Edit Metadata'].download_metadata(ids_add_metadata, confirm=False, clear=msprefs['auto_clear'], identify=dl_identify, covers=dl_covers, callback_done=callback_done)
 
         self.input_queue.put((None, None, None))
 
+    def auto_convert(self):
+        from calibre.gui2.ui import get_gui
+        gui = get_gui()
+        gui.iactions['Convert Books'].auto_convert_auto_add(
+            self.auto_convert_books)
+            
     def start(self):
         try:
             id, opf, cover = self.input_queue.get_nowait()

=== modified file 'src/calibre/gui2/metadata/bulk_download.py'
--- src/calibre/gui2/metadata/bulk_download.py	2012-04-12 09:50:24 +0000
+++ src/calibre/gui2/metadata/bulk_download.py	2013-02-01 04:56:24 +0000
@@ -77,8 +77,8 @@
         l.setColumnStretch(1, 100)
 
         self.identify = self.covers = True
-        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
-        self.bb.rejected.connect(self.reject)
+        self.clear = False
+        self.bb = QDialogButtonBox()
         b = self.bb.addButton(_('Download only &metadata'),
                 self.bb.AcceptRole)
         b.clicked.connect(self.only_metadata)
@@ -87,20 +87,36 @@
                 self.bb.AcceptRole)
         b.clicked.connect(self.only_covers)
         b.setIcon(QIcon(I('default_cover.png')))
-        b = self.b = self.bb.addButton(_('&Configure download'), self.bb.ActionRole)
-        b.setIcon(QIcon(I('config.png')))
-        b.clicked.connect(partial(show_config, parent, self))
-        l.addWidget(self.bb, 1, 0, 1, 2)
-        b = self.bb.addButton(_('Download &both'),
-                self.bb.AcceptRole)
+        
+        self.bb2 = QDialogButtonBox()
+        b = self.bb2.addButton(_('Download &both'),
+                self.bb2.AcceptRole)
         b.clicked.connect(self.accept)
         b.setDefault(True)
         b.setAutoDefault(True)
         b.setIcon(QIcon(I('ok.png')))
+        b = self.bb2.addButton(_('Clear &IDs and download both'),
+                self.bb2.AcceptRole)
+        b.clicked.connect(self.clear_ids)
+        b.setIcon(QIcon(I('trash.png')))
 
+        self.bb3 = QDialogButtonBox(QDialogButtonBox.Cancel)
+        self.bb3.rejected.connect(self.reject)
+        b = self.b = self.bb3.addButton(_('&Configure download'), self.bb3.ActionRole)
+        b.setIcon(QIcon(I('config.png')))
+        b.clicked.connect(partial(show_config, parent, self))
+        
+        l.addWidget(self.bb, 1, 0, 1, 2, Qt.AlignHCenter)
+        l.addWidget(self.bb2, 2, 0, 1, 2, Qt.AlignHCenter)
+        l.addWidget(self.bb3, 3, 0, 1, 2, Qt.AlignHCenter)
+        
         self.resize(self.sizeHint())
         b.setFocus(Qt.OtherFocusReason)
 
+    def clear_ids(self):
+        self.clear = True
+        self.accept()
+        
     def only_metadata(self):
         self.covers = False
         self.accept()
@@ -118,20 +134,34 @@
         ids = ids[batch_size:]
     return ans
 
-def start_download(gui, ids, callback, ensure_fields=None):
-    d = ConfirmDialog(ids, gui)
-    ret = d.exec_()
-    d.b.clicked.disconnect()
-    if ret != d.Accepted:
-        return
+def start_download(gui, ids, callback, ensure_fields=None, confirm=True, clear=False, identify=True, covers=True, callback_done=None):
+    if confirm:
+        d = ConfirmDialog(ids, gui)
+        ret = d.exec_()
+        d.b.clicked.disconnect()
+        if ret != d.Accepted:
+            return
+        identify = d.identify
+        covers = d.covers
+        clear  = d.clear
     tf = PersistentTemporaryFile('_metadata_bulk.log')
     tf.close()
 
+    # clear ids
+    if clear:
+        for id in ids:
+            gui.current_db.clear_identifiers(id)
+    
     job = Job('metadata bulk download',
         _('Download metadata for %d books')%len(ids),
-        download, (ids, tf.name, gui.current_db, d.identify, d.covers,
+        download, (ids, tf.name, gui.current_db, identify, covers,
             ensure_fields), {}, callback)
     job.download_debug_log = tf.name
+    job.callback_done = callback_done
+    # save current metadata for modification check
+    job.old_metadata = {}
+    for id in ids:
+        job.old_metadata[id] = gui.current_db.get_metadata(id, index_is_id=True, get_cover=False)
     gui.job_manager.run_threaded_job(job)
     gui.status_bar.show_message(_('Metadata download started'), 3000)
 

=== modified file 'src/calibre/gui2/preferences/metadata_sources.py'
--- src/calibre/gui2/preferences/metadata_sources.py	2012-10-06 07:32:50 +0000
+++ src/calibre/gui2/preferences/metadata_sources.py	2013-02-01 04:56:24 +0000
@@ -290,6 +290,10 @@
 
     def genesis(self, gui):
         r = self.register
+        r('confirm', msprefs)
+        r('auto_add', msprefs)
+        r('auto_overwrite', msprefs)
+        r('auto_clear', msprefs)
         r('txt_comments', msprefs)
         r('max_tags', msprefs)
         r('wait_after_first_identify_result', msprefs)

=== modified file 'src/calibre/gui2/preferences/metadata_sources.ui'
--- src/calibre/gui2/preferences/metadata_sources.ui	2012-10-06 07:45:50 +0000
+++ src/calibre/gui2/preferences/metadata_sources.ui	2013-02-01 04:56:24 +0000
@@ -126,20 +126,48 @@
         </widget>
        </item>
        <item row="1" column="1" colspan="2">
+        <widget class="QCheckBox" name="opt_confirm">
+         <property name="text">
+          <string>Confirm before updating metadata in library</string>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="1" colspan="2">
+        <widget class="QCheckBox" name="opt_auto_add">
+         <property name="text">
+          <string>Automatically add missing metadata on import</string>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="1" colspan="2">
+        <widget class="QCheckBox" name="opt_auto_overwrite">
+         <property name="text">
+          <string>Automatically overwrite metadata on import</string>
+         </property>
+        </widget>
+       </item>
+       <item row="4" column="1" colspan="2">
+        <widget class="QCheckBox" name="opt_auto_clear">
+         <property name="text">
+          <string>Automatically clear IDs and overwrite metadata on import</string>
+         </property>
+        </widget>
+       </item>
+       <item row="5" column="1" colspan="2">
         <widget class="QCheckBox" name="opt_txt_comments">
          <property name="text">
           <string>Convert all downloaded comments to plain &amp;text</string>
          </property>
         </widget>
        </item>
-       <item row="2" column="1" colspan="2">
+       <item row="6" column="1" colspan="2">
         <widget class="QCheckBox" name="opt_swap_author_names">
          <property name="text">
           <string>Swap author names from FN LN to LN, FN</string>
          </property>
         </widget>
        </item>
-       <item row="5" column="1">
+       <item row="9" column="1">
         <widget class="QLabel" name="label_2">
          <property name="text">
           <string>Max. number of &amp;tags to download:</string>
@@ -149,10 +177,10 @@
          </property>
         </widget>
        </item>
-       <item row="5" column="2">
+       <item row="9" column="2">
         <widget class="QSpinBox" name="opt_max_tags"/>
        </item>
-       <item row="6" column="1">
+       <item row="10" column="1">
         <widget class="QLabel" name="label_3">
          <property name="text">
           <string>Max. &amp;time to wait after first match is found:</string>
@@ -162,14 +190,14 @@
          </property>
         </widget>
        </item>
-       <item row="6" column="2">
+       <item row="10" column="2">
         <widget class="QSpinBox" name="opt_wait_after_first_identify_result">
          <property name="suffix">
           <string> secs</string>
          </property>
         </widget>
        </item>
-       <item row="7" column="1">
+       <item row="11" column="1">
         <widget class="QLabel" name="label_4">
          <property name="text">
           <string>Max. time to wait after first &amp;cover is found:</string>
@@ -179,14 +207,14 @@
          </property>
         </widget>
        </item>
-       <item row="7" column="2">
+       <item row="11" column="2">
         <widget class="QSpinBox" name="opt_wait_after_first_cover_result">
          <property name="suffix">
           <string> secs</string>
          </property>
         </widget>
        </item>
-       <item row="4" column="1" colspan="2">
+       <item row="8" column="1" colspan="2">
         <widget class="QCheckBox" name="opt_fewer_tags">
          <property name="toolTip">
           <string>&lt;p&gt;Different metadata sources have different sets of tags for the same book. If this option is checked, then calibre will use the smaller tag sets. These tend to be more like genres, while the larger tag sets tend to describe the books content.
@@ -197,7 +225,7 @@
          </property>
         </widget>
        </item>
-       <item row="3" column="1" colspan="2">
+       <item row="7" column="1" colspan="2">
         <widget class="QCheckBox" name="opt_find_first_edition_date">
          <property name="text">
           <string>Use published date of &quot;first edition&quot; (from worldcat.org)</string>

=== modified file 'src/calibre/library/database2.py'
--- src/calibre/library/database2.py	2013-01-29 06:26:38 +0000
+++ src/calibre/library/database2.py	2013-02-01 04:56:24 +0000
@@ -532,6 +532,10 @@
             self.refresh()
             self.refresh_format_cache()
         self.last_update_check = utcnow()
+        
+    def check_if_standard_metadata_modified(self, index, index_is_id, other):
+        mi = self.get_metadata(index, index_is_id=index_is_id, get_cover=False)
+        return mi.get_all_standard_metadata(True) != other.get_all_standard_metadata(True)
 
     def path(self, index, index_is_id=False):
         'Return the relative path to the directory containing this books files as a unicode string.'
@@ -1504,6 +1508,7 @@
         self.refresh_ids([id])
         if notify:
             self.notify('metadata', [id])
+        run_plugins_on_postimport(self, id, format)
         return True
 
     def save_original_format(self, book_id, fmt, notify=True):
@@ -3296,6 +3301,9 @@
         if notify:
             self.notify('metadata', [id_])
 
+    def clear_identifiers(self, id_):
+        self.set_identifiers(id_, {})
+            
     def set_isbn(self, id_, isbn, notify=True, commit=True):
         self.set_identifier(id_, 'isbn', isbn, notify=notify, commit=commit)