diff --git a/.gitignore b/.gitignore index f3f3d471..f4e159f8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,7 @@ # Cache /DeDRM_plugin/__pycache__ -/DeDRM_plugin/standalone/__pycache__ \ No newline at end of file +/DeDRM_plugin/standalone/__pycache__ +__pycache__/ +*.pyc +.pytest_cache/ \ No newline at end of file diff --git a/Obok_DeDRM_plugin.zip b/Obok_DeDRM_plugin.zip new file mode 100644 index 00000000..f18c4d7d Binary files /dev/null and b/Obok_DeDRM_plugin.zip differ diff --git a/Obok_plugin/action.py b/Obok_plugin/action.py index 36562af5..b8af3af9 100644 --- a/Obok_plugin/action.py +++ b/Obok_plugin/action.py @@ -34,6 +34,7 @@ from calibre_plugins.obok_dedrm.obok.obok import KoboLibrary from calibre_plugins.obok_dedrm.obok.legacy_obok import legacy_obok +from calibre_plugins.obok_dedrm.title_match import LibraryMatcher PLUGIN_ICONS = ['images/obok.png'] @@ -131,8 +132,32 @@ def launchObok(self): showErrorDlg(msg, None) return + # Build a matcher over the titles already in the calibre library so the + # dialog can flag (and filter) Kobo books that are already imported. + # Kobo and calibre store titles independently, so LibraryMatcher + # normalizes both sides and falls back to a bounded fuzzy match. + try: + book_ids = self.db.all_book_ids() + entries = [] + for book_id in book_ids: + title = self.db.field_for('title', book_id) + authors = self.db.field_for('authors', book_id) + if isinstance(authors, (tuple, list)): + author = ' & '.join(a for a in authors if a) + else: + author = authors or '' + entries.append((title, author)) + matcher = LibraryMatcher(entries) + debug_print("OBOK DIAG: built library matcher from %d calibre books (%d unique normalized titles)" + % (len(book_ids), len(matcher))) + except Exception: + debug_print("OBOK DIAG: FAILED to read calibre library titles -- traceback follows") + traceback.print_exc() + matcher = LibraryMatcher([]) + # Launch the Dialog so the user can select titles. - dlg = SelectionDialog(self.gui, self, books) + from calibre_plugins.obok_dedrm.config import plugin_prefs + dlg = SelectionDialog(self.gui, self, books, plugin_prefs, matcher) if dlg.exec_(): books_to_import = dlg.getBooks() self.count = len(books_to_import) diff --git a/Obok_plugin/config.py b/Obok_plugin/config.py index fdfb424b..64f32995 100644 --- a/Obok_plugin/config.py +++ b/Obok_plugin/config.py @@ -13,6 +13,7 @@ plugin_prefs.defaults['finding_homes_for_formats'] = 'Ask' plugin_prefs.defaults['kobo_serials'] = [] plugin_prefs.defaults['kobo_directory'] = u'' +plugin_prefs.defaults['hidden_books'] = [] from calibre_plugins.obok_dedrm.__init__ import PLUGIN_NAME, PLUGIN_VERSION from calibre_plugins.obok_dedrm.utilities import (debug_print) diff --git a/Obok_plugin/dialogs.py b/Obok_plugin/dialogs.py index 16bfcb08..9e9d81e2 100644 --- a/Obok_plugin/dialogs.py +++ b/Obok_plugin/dialogs.py @@ -9,19 +9,22 @@ LAB_DRM_FREE = '* : drm - free' try: - from PyQt5.Qt import (Qt, QVBoxLayout, QLabel, QApplication, QGroupBox, - QDialogButtonBox, QHBoxLayout, QTextBrowser, QProgressDialog, - QTimer, QSize, QDialog, QIcon, QTableWidget, QTableWidgetItem) + from PyQt5.Qt import (Qt, QVBoxLayout, QLabel, QApplication, QGroupBox, + QDialogButtonBox, QHBoxLayout, QTextBrowser, QProgressDialog, + QTimer, QSize, QDialog, QIcon, QTableWidget, QTableWidgetItem, + QLineEdit) except ImportError: - from PyQt4.Qt import (Qt, QVBoxLayout, QLabel, QApplication, QGroupBox, - QDialogButtonBox, QHBoxLayout, QTextBrowser, QProgressDialog, - QTimer, QSize, QDialog, QIcon, QTableWidget, QTableWidgetItem) + from PyQt4.Qt import (Qt, QVBoxLayout, QLabel, QApplication, QGroupBox, + QDialogButtonBox, QHBoxLayout, QTextBrowser, QProgressDialog, + QTimer, QSize, QDialog, QIcon, QTableWidget, QTableWidgetItem, + QLineEdit) try: from PyQt5.QtWidgets import (QListWidget, QAbstractItemView) except ImportError: from PyQt4.QtGui import (QListWidget, QAbstractItemView) +from calibre.constants import DEBUG from calibre.gui2 import gprefs, warning_dialog, error_dialog from calibre.gui2.dialogs.message_box import MessageBox @@ -32,6 +35,7 @@ ) from calibre_plugins.obok_dedrm.__init__ import (PLUGIN_NAME, PLUGIN_SAFE_NAME, PLUGIN_VERSION, PLUGIN_DESCRIPTION) +from calibre_plugins.obok_dedrm.title_match import LibraryMatcher, title_keys try: debug_print("obok::dialogs.py - loading translations") @@ -44,17 +48,35 @@ class SelectionDialog(SizePersistedDialog): ''' Dialog to select the kobo books to decrypt ''' - def __init__(self, gui, interface_action, books): + def __init__(self, gui, interface_action, books, plugin_prefs, matcher=None): ''' :param gui: Parent gui :param interface_action: InterfaceActionObject (InterfacePluginAction class from action.py) :param books: list of Kobo book + :param plugin_prefs: JSONConfig plugin preferences (for persisting hidden books) + :param matcher: LibraryMatcher over titles already in the calibre library ''' - + self.books = books self.gui = gui self.interface_action = interface_action - self.books = books + self.plugin_prefs = plugin_prefs + self.hidden_book_ids = set(plugin_prefs.get('hidden_books', [])) + self.matcher = matcher if matcher is not None else LibraryMatcher([]) + # Default to the import-eligible diff (books not yet in calibre); the + # 'Not in Library' button below is checked to match this on open. + self.library_filter_mode = 'not_in' # None | 'in' | 'not_in' + + # Precompute, once, which Kobo books are already in the calibre library. + # Done here -- not in apply_filters -- so the fuzzy scan does not re-run + # on every keystroke / filter toggle. + self.in_library_ids = set() + for book in self.books: + if self.matcher.match(book.title, book.author) is not None: + self.in_library_ids.add(book.volumeid) + + if DEBUG: + self._log_match_diagnostics() SizePersistedDialog.__init__(self, gui, PLUGIN_NAME + 'plugin:selections dialog') self.setWindowTitle(_(PLUGIN_NAME + ' v' + PLUGIN_VERSION)) @@ -73,12 +95,18 @@ def __init__(self, gui, interface_action, books): title_layout.setAlignment(Qt.AlignTop) layout.addSpacing(5) + + # Search box + search_layout = QHBoxLayout() + search_layout.addWidget(QLabel(_('Search:'), self)) + self.search_box = QLineEdit(self) + self.search_box.setPlaceholderText(_('Filter by title, author, or series...')) + self.search_box.textChanged.connect(self._apply_filters) + search_layout.addWidget(self.search_box) + layout.addLayout(search_layout) + main_layout = QHBoxLayout() layout.addLayout(main_layout) -# self.listy = QListWidget() -# self.listy.setSelectionMode(QAbstractItemView.ExtendedSelection) -# main_layout.addWidget(self.listy) -# self.listy.addItems(books) self.books_table = BookListTableWidget(self) main_layout.addWidget(self.books_table) @@ -87,28 +115,104 @@ def __init__(self, gui, interface_action, books): button_box.accepted.connect(self._ok_clicked) button_box.rejected.connect(self.reject) self.select_all_button = button_box.addButton(_("Select All"), QDialogButtonBox.ResetRole) - self.select_all_button.setToolTip(_("Select all books to add them to the calibre library.")) + self.select_all_button.setToolTip(_("Select all visible books.")) self.select_all_button.clicked.connect(self._select_all_clicked) - self.select_drm_button = button_box.addButton(_("All with DRM"), QDialogButtonBox.ResetRole) - self.select_drm_button.setToolTip(_("Select all books with DRM.")) - self.select_drm_button.clicked.connect(self._select_drm_clicked) - self.select_free_button = button_box.addButton(_("All DRM free"), QDialogButtonBox.ResetRole) - self.select_free_button.setToolTip(_("Select all books without DRM.")) - self.select_free_button.clicked.connect(self._select_free_clicked) + self.deselect_all_button = button_box.addButton(_("Deselect All"), QDialogButtonBox.ResetRole) + self.deselect_all_button.setToolTip(_("Deselect all books.")) + self.deselect_all_button.clicked.connect(self._deselect_all_clicked) + self.hide_button = button_box.addButton(_("Hide Selected"), QDialogButtonBox.ResetRole) + self.hide_button.setToolTip(_("Hide checked books from this list. They won't appear next time.")) + self.hide_button.clicked.connect(self._hide_selected_clicked) + self.show_hidden_button = button_box.addButton(_("Show Hidden"), QDialogButtonBox.ResetRole) + self.show_hidden_button.setToolTip(_("Show all previously hidden books.")) + self.show_hidden_button.clicked.connect(self._show_hidden_clicked) + self.in_library_button = button_box.addButton(_("In Library"), QDialogButtonBox.ResetRole) + self.in_library_button.setToolTip(_("Toggle: show only books already in your calibre library.")) + self.in_library_button.setCheckable(True) + self.in_library_button.clicked.connect(self._in_library_clicked) + self.not_in_library_button = button_box.addButton(_("Not in Library"), QDialogButtonBox.ResetRole) + self.not_in_library_button.setToolTip(_("Toggle: show only books NOT yet in your calibre library (eligible to import).")) + self.not_in_library_button.setCheckable(True) + self.not_in_library_button.clicked.connect(self._not_in_library_clicked) + self.not_in_library_button.setChecked(True) # default view: not-in-library layout.addWidget(button_box) # Cause our dialog size to be restored from prefs or created on first usage self.resize_dialog() self.books_table.populate_table(self.books) + self._apply_filters() + + def _log_match_diagnostics(self): + ''' + Verbose in-library match breakdown. Only called when calibre is run with + DEBUG on (e.g. ``calibre-debug -g``); silent and skipped in normal use. + ''' + in_lib_exact = in_lib_approx = 0 + author_mismatch = [] + for book in self.books: + hit = self.matcher.match(book.title, book.author) + if hit is not None: + if hit in title_keys(book.title): + in_lib_exact += 1 + else: + in_lib_approx += 1 + debug_print("OBOK DIAG: approximate in-library match: %r (%r) ~= calibre %r" + % (book.title, book.author, hit)) + elif self.matcher.title_exists(book.title): + # Title is in the library but the author didn't match: the + # author-first gate is treating it as a different book. + author_mismatch.append((book.title, book.author)) + debug_print("OBOK DIAG: %d Kobo books; in-library exact=%d approx=%d; not-in-library=%d" + % (len(self.books), in_lib_exact, in_lib_approx, + len(self.books) - len(self.in_library_ids))) + if author_mismatch: + debug_print("OBOK DIAG: %d not-in-library books whose TITLE exists under a different author:" + % len(author_mismatch)) + for t, a in author_mismatch[:15]: + debug_print("OBOK DIAG: author-mismatch: %r (Kobo author: %r)" % (t, a)) + + def _apply_filters(self): + search_text = self.search_box.text().lower() + self.books_table.apply_filters(search_text, self.hidden_book_ids, + self.library_filter_mode, self.in_library_ids) + + def _in_library_clicked(self): + # 'In Library' and 'Not in Library' are mutually exclusive views. + if self.in_library_button.isChecked(): + self.not_in_library_button.setChecked(False) + self.library_filter_mode = 'in' + else: + self.library_filter_mode = None + self._apply_filters() + + def _not_in_library_clicked(self): + if self.not_in_library_button.isChecked(): + self.in_library_button.setChecked(False) + self.library_filter_mode = 'not_in' + else: + self.library_filter_mode = None + self._apply_filters() + + def _save_hidden_books(self): + self.plugin_prefs['hidden_books'] = list(self.hidden_book_ids) + + def _hide_selected_clicked(self): + new_ids = self.books_table.get_checked_volumeids() + self.hidden_book_ids.update(new_ids) + self._save_hidden_books() + self.books_table.deselect_all() + self._apply_filters() + + def _show_hidden_clicked(self): + self.hidden_book_ids.clear() + self._save_hidden_books() + self._apply_filters() def _select_all_clicked(self): self.books_table.select_all() - def _select_drm_clicked(self): - self.books_table.select_drm(True) - - def _select_free_clicked(self): - self.books_table.select_drm(False) + def _deselect_all_clicked(self): + self.books_table.deselect_all() def _help_link_activated(self, url): ''' @@ -120,7 +224,12 @@ def _ok_clicked(self): ''' Build an index of the selected titles ''' - if len(self.books_table.selectedItems()): + books = self.books_table.get_books() + if len(books): + # Auto-hide the books we're about to import + for book in books: + self.hidden_book_ids.add(book.volumeid) + self._save_hidden_books() self.accept() else: msg = 'You must make a selection!' @@ -137,13 +246,14 @@ class BookListTableWidget(QTableWidget): def __init__(self, parent): QTableWidget.__init__(self, parent) + self.setSelectionMode(QAbstractItemView.SingleSelection) self.setSelectionBehavior(QAbstractItemView.SelectRows) def populate_table(self, books): self.clear() self.setAlternatingRowColors(True) self.setRowCount(len(books)) - header_labels = ['DRM', _('Title'), _('Author'), _('Series'), 'book_id'] + header_labels = ['', 'DRM', _('Title'), _('Author'), _('Series'), 'book_id'] self.setColumnCount(len(header_labels)) self.setHorizontalHeaderLabels(header_labels) self.verticalHeader().setDefaultSectionSize(24) @@ -156,12 +266,11 @@ def populate_table(self, books): self.setSortingEnabled(False) self.resizeColumnsToContents() - self.setMinimumColumnWidth(1, 100) + self.setColumnWidth(0, 30) self.setMinimumColumnWidth(2, 100) + self.setMinimumColumnWidth(3, 100) self.setMinimumSize(300, 0) - if len(books) > 0: - self.selectRow(0) - self.hideColumn(4) + self.hideColumn(5) self.setSortingEnabled(True) def setMinimumColumnWidth(self, col, minimum): @@ -169,53 +278,98 @@ def setMinimumColumnWidth(self, col, minimum): self.setColumnWidth(col, minimum) def populate_table_row(self, row, book): + # Column 0: Checkbox + check_item = QTableWidgetItem() + check_item.setFlags(Qt.ItemIsUserCheckable | Qt.ItemIsEnabled) + check_item.setCheckState(Qt.Unchecked) + check_item.setData(Qt.UserRole, book.volumeid) + self.setItem(row, 0, check_item) + + # Column 1: DRM icon if book.has_drm: icon = get_icon('drm-locked.png') val = 1 else: icon = get_icon('drm-unlocked.png') val = 0 - status_cell = IconWidgetItem(None, icon, val) status_cell.setData(Qt.UserRole, val) - self.setItem(row, 0, status_cell) - self.setItem(row, 1, ReadOnlyTableWidgetItem(book.title)) - self.setItem(row, 2, AuthorTableWidgetItem(book.author, book.author)) - self.setItem(row, 3, SeriesTableWidgetItem(book.series, book.series_index)) - self.setItem(row, 4, NumericTableWidgetItem(row)) + self.setItem(row, 1, status_cell) + + # Column 2: Title (store lowercased title in UserRole for search filtering) + title_item = ReadOnlyTableWidgetItem(book.title) + title_item.setData(Qt.UserRole, book.title.lower()) + self.setItem(row, 2, title_item) + + # Column 3: Author + self.setItem(row, 3, AuthorTableWidgetItem(book.author, book.author)) + + # Column 4: Series + self.setItem(row, 4, SeriesTableWidgetItem(book.series, book.series_index)) + + # Column 5: book_id (hidden) + self.setItem(row, 5, NumericTableWidgetItem(row)) + + def keyPressEvent(self, event): + if event.key() == Qt.Key_Space: + row = self.currentRow() + if row >= 0 and not self.isRowHidden(row): + item = self.item(row, 0) + if item.checkState() == Qt.Checked: + item.setCheckState(Qt.Unchecked) + else: + item.setCheckState(Qt.Checked) + else: + QTableWidget.keyPressEvent(self, event) def get_books(self): -# debug_print("BookListTableWidget:get_books - self.books:", self.books) books = [] - if len(self.selectedItems()): - for row in range(self.rowCount()): -# debug_print("BookListTableWidget:get_books - row:", row) - if self.item(row, 0).isSelected(): - book_num = convert_qvariant(self.item(row, 4).data(Qt.DisplayRole)) - debug_print("BookListTableWidget:get_books - book_num:", book_num) - book = self.books[book_num] - debug_print("BookListTableWidget:get_books - book:", book.title) - books.append(book) + for row in range(self.rowCount()): + if self.item(row, 0).checkState() == Qt.Checked: + book_num = convert_qvariant(self.item(row, 5).data(Qt.DisplayRole)) + book = self.books[book_num] + books.append(book) return books + def get_checked_volumeids(self): + ids = set() + for row in range(self.rowCount()): + if self.item(row, 0).checkState() == Qt.Checked: + ids.add(self.item(row, 0).data(Qt.UserRole)) + return ids + def select_all(self): - self .selectAll() + for row in range(self.rowCount()): + if not self.isRowHidden(row): + self.item(row, 0).setCheckState(Qt.Checked) - def select_drm(self, has_drm): - self.clearSelection() - current_selection_mode = self.selectionMode() - self.setSelectionMode(QAbstractItemView.MultiSelection) + def deselect_all(self): for row in range(self.rowCount()): -# debug_print("BookListTableWidget:select_drm - row:", row) - if convert_qvariant(self.item(row, 0).data(Qt.UserRole)) == 1: -# debug_print("BookListTableWidget:select_drm - has DRM:", row) - if has_drm: - self.selectRow(row) + self.item(row, 0).setCheckState(Qt.Unchecked) + + def apply_filters(self, search_text, hidden_book_ids, library_filter_mode=None, in_library_ids=None): + in_library_ids = in_library_ids or set() + for row in range(self.rowCount()): + volumeid = self.item(row, 0).data(Qt.UserRole) + title_lower = self.item(row, 2).data(Qt.UserRole) + author_text = self.item(row, 3).text().lower() if self.item(row, 3).text() else '' + series_text = self.item(row, 4).text().lower() if self.item(row, 4).text() else '' + + hidden_by_search = bool(search_text) and ( + search_text not in title_lower and + search_text not in author_text and + search_text not in series_text + ) + hidden_by_user = volumeid in hidden_book_ids + in_lib = volumeid in in_library_ids + if library_filter_mode == 'in': + hidden_by_library = not in_lib + elif library_filter_mode == 'not_in': + hidden_by_library = in_lib else: -# debug_print("BookListTableWidget:select_drm - DRM free:", row) - if not has_drm: - self.selectRow(row) - self.setSelectionMode(current_selection_mode) + hidden_by_library = False + + self.setRowHidden(row, hidden_by_search or hidden_by_user or hidden_by_library) class DecryptAddProgressDialog(QProgressDialog): diff --git a/Obok_plugin/obok/obok.py b/Obok_plugin/obok/obok.py index a8379b1c..44480fd3 100644 --- a/Obok_plugin/obok/obok.py +++ b/Obok_plugin/obok/obok.py @@ -429,7 +429,7 @@ def __getmacaddrs (self): """The list of all MAC addresses on this machine.""" macaddrs = [] if sys.platform.startswith('win'): - c = re.compile('\s?(' + '[0-9a-f]{2}[:\-]' * 5 + '[0-9a-f]{2})(\s|$)', re.IGNORECASE) + c = re.compile(r'\s?(' + r'[0-9a-f]{2}[:\-]' * 5 + r'[0-9a-f]{2})(\s|$)', re.IGNORECASE) try: output = subprocess.Popen('ipconfig /all', shell=True, stdout=subprocess.PIPE, text=True).stdout for line in output: @@ -443,7 +443,7 @@ def __getmacaddrs (self): if m: macaddrs.append(re.sub("-", ":", m.group(1)).upper()) elif sys.platform.startswith('darwin'): - c = re.compile('\s(' + '[0-9a-f]{2}:' * 5 + '[0-9a-f]{2})(\s|$)', re.IGNORECASE) + c = re.compile(r'\s(' + '[0-9a-f]{2}:' * 5 + r'[0-9a-f]{2})(\s|$)', re.IGNORECASE) output = subprocess.check_output('/sbin/ifconfig -a', shell=True, encoding='utf-8') matches = c.findall(output) for m in matches: @@ -459,14 +459,14 @@ def __getmacaddrs (self): else: # final fallback # let's try ip - c = re.compile('\s(' + '[0-9a-f]{2}:' * 5 + '[0-9a-f]{2})(\s|$)', re.IGNORECASE) + c = re.compile(r'\s(' + '[0-9a-f]{2}:' * 5 + r'[0-9a-f]{2})(\s|$)', re.IGNORECASE) for line in os.popen('ip -br link'): m = c.search(line) if m: macaddrs.append(m.group(1).upper()) # let's try ipconfig under wine - c = re.compile('\s(' + '[0-9a-f]{2}-' * 5 + '[0-9a-f]{2})(\s|$)', re.IGNORECASE) + c = re.compile(r'\s(' + '[0-9a-f]{2}-' * 5 + r'[0-9a-f]{2})(\s|$)', re.IGNORECASE) for line in os.popen('ipconfig /all'): m = c.search(line) if m: @@ -665,7 +665,7 @@ def decrypt_book(book, lib): print("Converting {0}".format(book.title)) zin = zipfile.ZipFile(book.filename, "r") # make filename out of Unicode alphanumeric and whitespace equivalents from title - outname = "{0}.epub".format(re.sub('[^\s\w]', '_', book.title, 0, re.UNICODE)) + outname = "{0}.epub".format(re.sub(r'[^\s\w]', '_', book.title, 0, re.UNICODE)) if (book.type == 'drm-free'): print("DRM-free book, conversion is not needed") shutil.copyfile(book.filename, outname) diff --git a/Obok_plugin/title_match.py b/Obok_plugin/title_match.py new file mode 100644 index 00000000..7c7c0167 --- /dev/null +++ b/Obok_plugin/title_match.py @@ -0,0 +1,343 @@ +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__docformat__ = 'restructuredtext en' + +''' +Title matching between Kobo books and the calibre library. + +Kobo and calibre maintain a book's title independently, so a plain exact +string compare misses a large fraction of real duplicates: + + * smart vs straight quotes/apostrophes ("All the King's Men") + * a subtitle present on one side only ("Born a Crime: Stories ...") + * a trailing parenthetical / edition tag ("... (Pulitzer Prize Winner)", + "... 10th Anniversary") + * "&" vs "and" + +`normalize_title` canonicalizes a title so the two sides can be compared +reliably; it MUST be applied identically to both sides. `LibraryMatcher` +wraps the calibre side and, when the normalized forms don't match exactly, +falls back to a bounded Levenshtein (edit-distance) similarity check to catch +typos and minor variations. + +This module deliberately has no calibre / Qt imports so it can be unit tested +on its own (see tests/test_title_match.py). +''' + +import re +import unicodedata + +# Punctuation that differs between Kobo and calibre but should be treated as +# equivalent. Folded before the (lossy) generic punctuation pass below so that +# colon/paren detection still works on already-folded text. +_QUOTE_FOLDS = { + '‘': "'", '’': "'", '‚': "'", '‛': "'", + '´': "'", '`': "'", + '“': '"', '”': '"', '„': '"', '‟': '"', + '–': '-', '—': '-', '‒': '-', '―': '-', '−': '-', +} + +# A trailing parenthetical/bracketed group, e.g. " (Pulitzer Prize Winner)". +_TRAILING_BRACKET_RE = re.compile(r'\s*[\(\[\{][^\(\)\[\]\{\}]*[\)\]\}]\s*$') + +# A trailing edition descriptor, e.g. " 10th Anniversary", " Deluxe Edition". +# "anniversary" may stand alone (rarely a real title word); the others must be +# followed by "edition" to avoid stripping legitimate title words. +_EDITION_SUFFIX_RE = re.compile( + r'\s+(?:' + r'(?:\d+\s*(?:st|nd|rd|th)\s+)?anniversary(?:\s+edition)?' + r'|(?:deluxe|revised|expanded|updated|illustrated|annotated|special' + r'|definitive|collector\'?s?|unabridged|abridged|reprint)\s+edition' + r')\s*$' +) + +_NON_ALNUM_RE = re.compile(r'[^\w\s]', re.UNICODE) +_WHITESPACE_RE = re.compile(r'\s+') + +# An English article at the front ("The Hobbit") or moved to the end in +# calibre's sort form ("Hobbit, The", which normalizes to a trailing " the"). +_LEADING_ARTICLE_RE = re.compile(r'^(?:the|a|an)\s+') +_TRAILING_ARTICLE_RE = re.compile(r'\s+(?:the|a|an)$') + +# Fuzzy-match tuning. Kept as module constants so they're trivial to adjust +# from observed behaviour (the dialog logs every fuzzy match it makes). +FUZZY_THRESHOLD = 0.87 # minimum similarity (1.0 == identical) to accept +FUZZY_MIN_LEN = 6 # don't fuzzy-match titles shorter than this + + +def normalize_title(title, strip_subtitle=True): + ''' + Canonicalize a book title for comparison. Returns a lowercased, punctuation + -folded string with parenthetical / edition noise removed. When + `strip_subtitle` is true (default) the part after the first colon is also + dropped. Prefer `title_keys` for matching, which considers both forms. + ''' + if not title: + return '' + try: + s = unicodedata.normalize('NFKC', title) + except TypeError: + # py2 bytes safety net; modern calibre passes unicode. + s = unicodedata.normalize('NFKC', title.decode('utf-8', 'replace')) + for src, dst in _QUOTE_FOLDS.items(): + if src in s: + s = s.replace(src, dst) + s = s.lower() + s = s.replace('&', ' and ') + if strip_subtitle: + # Drop subtitle after the first colon ("Born a Crime: Stories ..." -> ...). + s = s.split(':', 1)[0] + # Strip trailing parenthetical/bracketed groups (possibly nested/repeated). + prev = None + while prev != s: + prev = s + s = _TRAILING_BRACKET_RE.sub('', s) + # Strip a trailing edition descriptor. + s = _EDITION_SUFFIX_RE.sub('', s) + # Fold any remaining punctuation (apostrophes, hyphens, %, ...) to spaces. + s = _NON_ALNUM_RE.sub(' ', s) + s = _WHITESPACE_RE.sub(' ', s).strip() + return s + + +def title_keys(title): + ''' + The set of normalized forms a title can match on: the subtitle-stripped form + AND the full form (colon kept as a separator). Comparing the sets on both + sides means "Foo: Bar" matches a bare "Foo" *and* a colon-less "Foo Bar". + ''' + keys = set() + stripped = normalize_title(title, strip_subtitle=True) + if stripped: + keys.add(stripped) + full = normalize_title(title, strip_subtitle=False) + if full: + keys.add(full) + return keys + + +def _article_key(normalized): + ''' + Reduce an already-normalized title to an article-insensitive key by removing + an article at the front ("the hobbit") or the end (calibre's "hobbit, the", + which normalizes to a trailing " the"). + ''' + s = _LEADING_ARTICLE_RE.sub('', normalized) + s = _TRAILING_ARTICLE_RE.sub('', s) + return s.strip() + + +def normalize_author(author): + ''' + Canonicalize an author for comparison. Lowercases, drops punctuation, and + sorts the name tokens so display order / "Last, First" don't matter + ("Robert Penn Warren" == "Warren, Robert Penn"). Returns '' if unknown. + ''' + if not author: + return '' + try: + s = unicodedata.normalize('NFKC', author) + except TypeError: + s = unicodedata.normalize('NFKC', author.decode('utf-8', 'replace')) + s = s.lower() + s = _NON_ALNUM_RE.sub(' ', s) + return ' '.join(sorted(s.split())) + + +def _strong_tokens(na): + ''' + The distinctive tokens of a normalized author string: alphabetic name parts + of length >= 3. Drops single initials ("j") and purely numeric junk (an + ISBN or year that crept into the author field, as calibre sometimes stores). + ''' + return frozenset(t for t in na.split() if len(t) >= 3 and not t.isdigit()) + + +def _bounded_levenshtein(a, b, max_dist): + ''' + Levenshtein edit distance between `a` and `b`, but returns `max_dist + 1` + as soon as the distance is guaranteed to exceed `max_dist`. The early-out + keeps the per-comparison cost low when scanning a large library. + ''' + la, lb = len(a), len(b) + if abs(la - lb) > max_dist: + return max_dist + 1 + if la == 0: + return lb + if lb == 0: + return la + if la > lb: # keep the inner row short + a, b = b, a + la, lb = lb, la + previous = list(range(la + 1)) + for j in range(1, lb + 1): + current = [j] + [0] * la + bj = b[j - 1] + row_min = current[0] + for i in range(1, la + 1): + cost = 0 if a[i - 1] == bj else 1 + current[i] = min(previous[i] + 1, + current[i - 1] + 1, + previous[i - 1] + cost) + if current[i] < row_min: + row_min = current[i] + if row_min > max_dist: + return max_dist + 1 + previous = current + return previous[la] + + +class _AuthorGroup(object): + '''Title keys for all library books by one (normalized) author.''' + __slots__ = ('titles', 'article') + + def __init__(self): + self.titles = set() # every title key by this author + self.article = {} # article key -> title key + + def add(self, keys): + self.titles.update(keys) + for k in keys: + self.article[_article_key(k)] = k + + +class LibraryMatcher(object): + ''' + Decides whether a (Kobo) book is already present in the calibre library. + + A book is the same when the AUTHOR(S) agree *and* the title harmonizes. + Author agreement is checked first: authors are normalized to a set of name + tokens, tolerant of order, "Last, First", separators, and one side listing + extra authors (subset/superset). Only within a matching author's books are + titles compared -- exact normalized form, then leading/trailing article, + then bounded fuzzy. A book with no usable author falls back to an exact + full-title match only (no fuzzy). Treat as immutable after construction. + ''' + + def __init__(self, entries, fuzzy=True, + threshold=FUZZY_THRESHOLD, min_len=FUZZY_MIN_LEN): + ''' + :param entries: iterable of calibre books; each item is either a title + string or a ``(title, author)`` pair. + ''' + self._fuzzy = fuzzy + self._threshold = threshold + self._min_len = min_len + self._all_titles = set() # every title key (for the no-author fallback) + self._groups = {} # normalized author -> _AuthorGroup + self._token_to_authors = {} # strong author token -> set of normalized authors + for entry in entries: + if isinstance(entry, (tuple, list)): + title = entry[0] + author = entry[1] if len(entry) > 1 else '' + else: + title, author = entry, '' + keys = title_keys(title) + if not keys: + continue + self._all_titles.update(keys) + na = normalize_author(author) + if not na: + continue + group = self._groups.get(na) + if group is None: + group = self._groups[na] = _AuthorGroup() + for tok in _strong_tokens(na): + self._token_to_authors.setdefault(tok, set()).add(na) + group.add(keys) + + def __len__(self): + return len(self._all_titles) + + def title_exists(self, title): + ''' + True if the title alone (ignoring author) matches some library book. + For diagnostics: title_exists() True while match() is None means the + title is present but under a different author. + ''' + return bool(title_keys(title) & self._all_titles) + + def contains(self, title, author=None): + '''True if `title` by `author` matches a book already in the library.''' + return self.match(title, author) is not None + + def match(self, title, author=None): + ''' + Return the normalized calibre title that `title` (by `author`) matches, + or None. Author is checked first; within a matching author, the title + is harmonized (exact form -> leading/trailing article -> bounded fuzzy). + (Returning the match -- not just a bool -- lets callers log *why* + something was considered a duplicate.) + ''' + keys = title_keys(title) + if not keys: + return None + na = normalize_author(author) + if not na: + # Author unknown: only trust an exact title match, nothing looser. + for k in keys: + if k in self._all_titles: + return k + return None + for group in self._candidate_groups(na): + # exact title within this author + for k in keys: + if k in group.titles: + return k + # same author, article-insensitive ("The X" / "X" / "X, The") + for k in keys: + hit = group.article.get(_article_key(k)) + if hit is not None: + return hit + # same author, bounded fuzzy title + hit = self._fuzzy_title(keys, group.titles) + if hit is not None: + return hit + return None + + def _candidate_groups(self, na): + ''' + Yield author groups that plausibly denote the same author as `na`: the + exact author first, then any author sharing a strong name token (see + `_strong_tokens`). This tolerates honorifics / ordination names ("Bhante" + vs "Henepola Gunaratana"), abbreviated middles ("Oren J." vs "Oren Jay"), + one side listing extra authors, and junk tokens like an ISBN calibre + stored as a second author. The caller's title check keeps it honest. + ''' + seen = set() + exact = self._groups.get(na) + if exact is not None: + seen.add(na) + yield exact + candidates = set() + for tok in _strong_tokens(na): + candidates |= self._token_to_authors.get(tok, set()) + for other_na in candidates: + if other_na in seen: + continue + seen.add(other_na) + yield self._groups[other_na] + + def _fuzzy_title(self, keys, titles): + if not self._fuzzy: + return None + n = max(keys, key=len) + if len(n) < self._min_len: + return None + ln = len(n) + threshold = self._threshold + for cand in titles: + lc = len(cand) + longest = ln if ln > lc else lc + if longest == 0: + continue + # Cheap upper bound on similarity from the length gap alone. + if (1.0 - (abs(ln - lc) / float(longest))) < threshold: + continue + max_dist = int((1.0 - threshold) * longest) + if _bounded_levenshtein(n, cand, max_dist) <= max_dist: + return cand + return None \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..477707b7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +# Obok_plugin/ is an importable package whose __init__.py needs the calibre +# runtime, so tests live at the repo root (outside any package) and import the +# pure title_match module directly via sys.path. +testpaths = tests +addopts = --import-mode=importlib diff --git a/tests/test_obok_title_match.py b/tests/test_obok_title_match.py new file mode 100644 index 00000000..d8bbd076 --- /dev/null +++ b/tests/test_obok_title_match.py @@ -0,0 +1,237 @@ +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +''' +Unit tests for title_match.normalize_title / LibraryMatcher. + +These use the real Kobo titles surfaced by the diagnostic build against +Erez's library (the ones that exact-matching missed), so the tests document +the actual failure modes we're fixing. + +Run from the repo root: pytest -v (or: python3 -m pytest -v) +''' +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +import os +import sys + +# title_match lives inside the Obok_plugin package dir; add it to the path so we +# can import the module without importing the package (whose __init__ needs calibre). +sys.path.insert(0, os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'Obok_plugin')) + +from title_match import normalize_title, normalize_author, LibraryMatcher # noqa: E402 + +CURLY = '’' # right single quotation mark (Kobo's apostrophe) + + +def n(t): + return normalize_title(t) + + +# --- normalize_title: the three observed drift buckets ----------------------- + +def test_smart_apostrophe_folds_to_straight(): + # Kobo stores a curly apostrophe; calibre a straight one. + assert n('All The King' + CURLY + 's Men') == n("All the King's Men") + assert n('Assassin' + CURLY + 's Apprentice') == n("Assassin's Apprentice") + assert n('Cat' + CURLY + 's Cradle') == n("Cat's Cradle") + + +def test_subtitle_after_colon_is_dropped(): + assert n('Born a Crime') == n('Born a Crime: Stories from a South African Childhood') + assert n('Be Useful') == n('Be Useful: Seven Tools for Life') + assert n('Co-Intelligence') == n('Co-Intelligence: Living and Working with AI') + + +def test_trailing_parenthetical_is_dropped(): + assert n('American Prometheus (Pulitzer Prize Winner)') == \ + n('American Prometheus: The Triumph and Tragedy of J. Robert Oppenheimer') + + +def test_parenthetical_and_subtitle_combined(): + assert n('Behold the Dreamers') == n("Behold the Dreamers (Oprah's Book Club): A Novel") + + +def test_trailing_edition_descriptor_is_dropped(): + assert n('10% Happier 10th Anniversary') == n('10% Happier') + assert n('Dune Deluxe Edition') == n('Dune') + + +def test_ampersand_equals_and(): + assert n('Crime & Punishment') == n('Crime and Punishment') + + +def test_distinct_titles_do_not_collide(): + assert n('The Hobbit') != n('The Silmarillion') + assert n('Dune') != n('Dune Messiah') + assert n('Be Useful') != n('Be Quiet') + + +# --- LibraryMatcher: exact-after-normalization ------------------------------- + +LIBRARY = [ + "All the King's Men", + 'Born a Crime: Stories from a South African Childhood', + 'Be Useful: Seven Tools for Life', + '10% Happier', + 'American Prometheus: The Triumph and Tragedy of J. Robert Oppenheimer', + 'The Hobbit', + 'The Lord of the Rings', +] + + +def test_matcher_finds_normalized_duplicates(): + m = LibraryMatcher(LIBRARY) + assert m.contains('All The King' + CURLY + 's Men') + assert m.contains('Born a Crime') + assert m.contains('Be Useful') + assert m.contains('10% Happier 10th Anniversary') + assert m.contains('American Prometheus (Pulitzer Prize Winner)') + + +def test_matcher_rejects_book_not_in_library(): + m = LibraryMatcher(LIBRARY) + assert not m.contains('A Wizard of Earthsea') + assert not m.contains('Assassin' + CURLY + 's Apprentice') + + +# --- LibraryMatcher: bounded fuzzy fallback ---------------------------------- + +def test_fuzzy_catches_minor_variation(): + # Fuzzy now runs only within a matching author. + m = LibraryMatcher([('The Lord of the Rings', 'J.R.R. Tolkien')]) + assert m.contains('The Lord of the Ring', 'J.R.R. Tolkien') # singular typo + assert m.contains('The Lord of teh Rings', 'J.R.R. Tolkien') # transposition + + +def test_fuzzy_disabled_for_short_titles(): + m = LibraryMatcher([('Dune', 'Frank Herbert')]) + assert not m.contains('June', 'Frank Herbert') # 1 edit away, too short to fuzzy + + +def test_fuzzy_threshold_rejects_different_book(): + m = LibraryMatcher([('The Hobbit', 'J.R.R. Tolkien')]) + assert not m.contains('The Rabbit', 'J.R.R. Tolkien') # similar but different book + + +def test_fuzzy_requires_matching_author(): + m = LibraryMatcher([('The Lord of the Rings', 'J.R.R. Tolkien')]) + assert not m.contains('The Lord of the Ring', 'Somebody Else') # author gate + + +# --- author-first behaviour -------------------------------------------------- + +def test_no_author_falls_back_to_exact_title_only(): + m = LibraryMatcher([('The Lord of the Rings', 'J.R.R. Tolkien')]) + assert m.contains('The Lord of the Rings') # exact title, no author -> ok + assert not m.contains('The Lord of the Ring') # looser matching needs an author + + +def test_author_subset_still_matches(): + # Kobo lists only one of the two co-authors; calibre lists both. + m = LibraryMatcher([('Good Omens', 'Terry Pratchett & Neil Gaiman')]) + assert m.contains('Good Omens', 'Neil Gaiman') + + +def test_same_title_different_author_is_not_a_match(): + # Title collision across authors must NOT match (the precision win). + m = LibraryMatcher([('Twilight', 'Stephenie Meyer')]) + assert not m.contains('Twilight', 'William Gay') + + +# --- real-world author drift (from Erez's library) --------------------------- + +def test_isbn_stored_as_second_author_is_ignored(): + # calibre stored an ISBN as a 2nd "author"; Kobo adds the illustrator. + m = LibraryMatcher([('Chloe and Cracker', 'Kelly McKain & 9781847153395')]) + assert m.contains('Chloe and Cracker', 'Kelly McKain, Mandy Stanley') + + +def test_honorific_vs_ordination_name_shares_surname(): + m = LibraryMatcher([('Mindfulness in Plain English', 'Henepola Gunaratana')]) + assert m.contains('Mindfulness in Plain English', 'Bhante Gunaratana') + + +def test_abbreviated_middle_name_and_subtitle(): + m = LibraryMatcher([('Say What You Mean: A Mindful Approach to Nonviolent ' + 'Communication', 'Oren J. Sofer')]) + assert m.contains('Say What You Mean', 'Oren Jay Sofer') + + +def test_shared_first_name_alone_does_not_match_different_titles(): + # Sharing only a first name is allowed as an author candidate, but the title + # gate still prevents merging genuinely different books. + m = LibraryMatcher([('The Firm', 'John Grisham')]) + assert not m.contains('A Perfect Spy', 'John le Carre') + + +def test_match_returns_the_matched_title(): + m = LibraryMatcher(['Born a Crime: Stories from a South African Childhood']) + assert m.match('Born a Crime') == normalize_title('Born a Crime') + assert m.match('Something Else Entirely') is None + + +# --- normalize_author -------------------------------------------------------- + +def test_author_order_and_format_are_ignored(): + # Flipped order and "Last, First" both reduce to the same key. + assert normalize_author('Grisham, John') == normalize_author('John Grisham') + assert normalize_author('Robert Penn Warren') == normalize_author('Warren, Robert Penn') + + +def test_distinct_authors_differ(): + assert normalize_author('John Grisham') != normalize_author('Stephen King') + + +# --- LibraryMatcher: same-author, article-insensitive match ------------------ + +def test_leading_article_match_when_author_agrees(): + m = LibraryMatcher([('The Hobbit', 'J.R.R. Tolkien')]) + assert m.contains('Hobbit', 'J.R.R. Tolkien') # Kobo dropped "The" + assert m.contains('Hobbit', 'Tolkien, J.R.R.') # + flipped author + + +def test_trailing_article_sort_form_matches(): + # calibre stores the sort form "..., A"; Kobo has the natural "A ...". + m = LibraryMatcher([('Guide for the Good Life, A', 'William Irvine')]) + assert m.contains('A Guide for the Good Life', 'William Irvine') + # ...and the reverse direction. + m2 = LibraryMatcher([('A Guide for the Good Life', 'Irvine, William')]) + assert m2.contains('Guide for the Good Life, A', 'William Irvine') + + +def test_article_match_requires_matching_author(): + m = LibraryMatcher([('The Hobbit', 'J.R.R. Tolkien')]) + assert not m.contains('Hobbit', 'Someone Else') # author gate blocks it + assert not m.contains('Hobbit') # no author -> no gate, no match + + +def test_article_match_does_not_merge_different_books_same_author(): + m = LibraryMatcher([('The Stand', 'Stephen King')]) + assert not m.contains('It', 'Stephen King') # same author, different book + + +# --- multi-author + colon-vs-no-colon (the New Yorker case) ------------------ + +def test_multi_author_order_and_separator_ignored(): + assert normalize_author('David Remnick & Bob Mankoff') == \ + normalize_author('Bob Mankoff, David Remnick') + + +def test_colon_in_one_title_only_still_matches(): + # Kobo writes the full title with a colon; calibre without one. + m = LibraryMatcher([('The New Yorker Encyclopedia of Cartoons', + 'Bob Mankoff, David Remnick')]) + assert m.contains('The New Yorker: Encyclopedia of Cartoons', + 'David Remnick & Bob Mankoff') + # ...and the reverse (colon on the calibre side). + m2 = LibraryMatcher([('The New Yorker: Encyclopedia of Cartoons', + 'Bob Mankoff & David Remnick')]) + assert m2.contains('The New Yorker Encyclopedia of Cartoons', + 'David Remnick, Bob Mankoff') + + +def test_colon_subtitle_only_on_one_side_still_matches_bare_title(): + # The original "Born a Crime" behaviour must still hold (stripped-form match). + m = LibraryMatcher(['Born a Crime: Stories from a South African Childhood']) + assert m.contains('Born a Crime')