diff --git a/action.py b/action.py index 73e09a2..ba0a895 100644 --- a/action.py +++ b/action.py @@ -11,7 +11,6 @@ import sys import importlib.util import time - from urllib.request import Request, urlopen from urllib.error import URLError, HTTPError @@ -50,6 +49,8 @@ KoreaderSync, ) +from calibre_plugins.koreader.md5_utils import partial_md5_checksum + from calibre.utils.iso8601 import utc_tz, local_tz from calibre.gui2.dialogs.message_box import MessageBox from calibre.gui2.actions import InterfaceAction @@ -63,6 +64,7 @@ from calibre.devices.usbms.driver import debug_print as root_debug_print from calibre.constants import numeric_version from enum import Enum, auto +from calibre.library import db __license__ = 'GNU GPLv3' __copyright__ = '2021, harmtemolder ' @@ -70,6 +72,7 @@ __modification_date__ = '2024' __docformat__ = 'restructuredtext en' + if numeric_version >= (5, 5, 0): module_debug_print = partial(root_debug_print, ' koreader:action:', sep='') else: @@ -376,6 +379,53 @@ def get_connected_device(self): def _on_device_metadata_available(self): self.sync_to_calibre(silent=True if not DEBUG else False) + def detect_hashdocsettings_path(self, device): + """Auto-detect the hashdocsettings path based on device type. + + :param device: a device object + :return: path to hashdocsettings directory or None if not found + """ + debug_print = partial( + module_debug_print, + 'KoreaderAction:detect_hashdocsettings_path:' + ) + + # Common hashdocsettings locations for different devices + possible_paths = [ + # Kobo + '../.adds/koreader/hashdocsettings', + '../../.adds/koreader/hashdocsettings', + # Kindle + '../koreader/hashdocsettings', + '../../koreader/hashdocsettings', + # PocketBook and others + '../system/koreader/hashdocsettings', + '../../system/koreader/hashdocsettings', + # Generic fallback + '../hashdocsettings', + '../.koreader/hashdocsettings', + ] + + # Try to find the directory that exists + for test_path in possible_paths: + debug_print(f'Testing path: {test_path}') + + test_file = test_path.replace('hashdocsettings', 'defaults.custom.lua') + + try: + # Try to guess hashdocsettings path with defaults.custom.lua + with io.BytesIO() as outfile: + device.get_file(test_file, outfile) + debug_print(f'Found: {test_file}') + return test_path + + except Exception as e: + debug_print(f'Error checking path {test_path}: {e}') + continue + + debug_print('Could not find hashdocsettings directory') + return None + def get_paths(self, device): """Retrieves paths to sidecars of all books in calibre's library on the device @@ -401,17 +451,58 @@ def get_paths(self, device): for book in device.books(): debug_print(f'uuid to path: {book.uuid} - {book.path}') - paths = { - book.uuid: re.sub( - r'\.(\w+)$', r'.sdr/metadata.\1.lua', book.path - ) - for book in device.books() - } + if CONFIG['checkbox_enable_sidecar_hashdocsettings']: + # Auto-detect hashdocsettings path if not manually set or if auto-detect is enabled + hashdocpath = CONFIG.get('sidecar_hashdocsettings_loc', '') + + # If path is empty or user wants auto-detection, try to find it + if not hashdocpath or CONFIG.get('checkbox_autodetect_hashdocsettings', True): + detected_path = self.detect_hashdocsettings_path(device) + if detected_path: + hashdocpath = detected_path + debug_print(f'Auto-detected hashdocsettings path: {hashdocpath}') + # Optionally save it to config for future use + if CONFIG.get('checkbox_save_detected_path', True): + CONFIG['sidecar_hashdocsettings_loc'] = hashdocpath + elif not hashdocpath: + # No path detected and no manual path set + debug_print('No hashdocsettings path found, falling back to standard sidecar location') + CONFIG['checkbox_enable_sidecar_hashdocsettings'] = False + # Fall through to standard path generation below + + if CONFIG['checkbox_enable_sidecar_hashdocsettings'] and hashdocpath: + paths = {} + for book in device.books(): + paths[book.uuid] = { + 'book_path': book.path, + 'needs_md5': True, + 'hashdocpath': hashdocpath + } + else: + # Fall back to standard path + paths = { + book.uuid: re.sub( + r'\.(\w+)$', r'.sdr/metadata.\1.lua', book.path + ) + for book in device.books() + } + else: + # Default path same as books + paths = { + book.uuid: re.sub( + r'\.(\w+)$', r'.sdr/metadata.\1.lua', book.path + ) + for book in device.books() + } - debug_print( - f'generated {len(paths)} path(s) to sidecar Lua files:\n\t', - '\n\t'.join(paths.values()) - ) + # Debug print only for standard paths + if not CONFIG.get('checkbox_enable_sidecar_hashdocsettings') or not any( + isinstance(p, dict) and p.get('needs_md5') for p in paths.values() + ): + debug_print( + f'generated {len(paths)} path(s) to sidecar Lua files:\n\t', + '\n\t'.join(str(p) for p in paths.values()) + ) return paths @@ -761,6 +852,38 @@ def sync_missing_sidecars_to_koreader(self, silent=False): sidecar_paths = self.get_paths(device) debug_print('sidecar_paths: ', sidecar_paths) + + # Resolve paths if using hashdocsettings + resolved_paths = {} + if CONFIG['checkbox_enable_sidecar_hashdocsettings']: + debug_print('Resolving paths for hashdocsettings...') + db = self.gui.current_db.new_api + + for book_uuid, path_info in sidecar_paths.items(): + if isinstance(path_info, dict) and path_info.get('needs_md5'): + try: + book_path = path_info['book_path'] + hashdocpath = path_info['hashdocpath'] + + # get partial MD5 + book_md5 = partial_md5_checksum(device, book_path) + + # Construire le chemin sidecar + extension = re.sub(r'.*\.(\w+)$', r'\1', book_path) + sidecar_path = ( + f"{hashdocpath}/{book_md5[:2]}/{book_md5}.sdr/" + f"metadata.{extension}.lua" + ) + resolved_paths[book_uuid] = sidecar_path + + except Exception as e: + debug_print(f'Error calculating partial MD5 for {book_uuid}: {e}') + continue + else: + resolved_paths[book_uuid] = path_info + + sidecar_paths = resolved_paths + sidecar_paths_exist = {} sidecar_paths_not_exist = {} for book_uuid, path in sidecar_paths.items(): @@ -1066,9 +1189,10 @@ def main(): main() # Runs scheduled_progress_sync + def sync_to_calibre(self, silent=False): - """This plugin’s main purpose. It syncs the contents of - KOReader’s metadata sidecar files into calibre’s metadata. + """This plugin's main purpose. It syncs the contents of + KOReader's metadata sidecar files into calibre's metadata. :return: """ @@ -1089,11 +1213,12 @@ class KOSyncWorker(QThread): progress_update = pyqtSignal(int, str) finished_signal = pyqtSignal(dict) - def __init__(self, action, db, sidecar_paths): + def __init__(self, action, db, sidecar_paths, device): super().__init__() self.action = action self.db = db self.sidecar_paths = sidecar_paths + self.device = device def run(self): results = [] @@ -1101,40 +1226,77 @@ def run(self): num_fail = 0 num_skip = 0 - for idx, (book_uuid, sidecar_path) in enumerate(sidecar_paths.items()): - debug_print('Trying to get sidecar from ', device, + # If ussing hashdocsettings, get partial MD5 first + if CONFIG['checkbox_enable_sidecar_hashdocsettings']: + debug_print('Calculating partial md5 hashes for hashdocsettings...') + resolved_paths = {} + + for book_uuid, path_info in self.sidecar_paths.items(): + if isinstance(path_info, dict) and path_info.get('needs_md5'): + try: + book_path = path_info['book_path'] + hashdocpath = path_info['hashdocpath'] + + book_id = self.db.lookup_by_uuid(book_uuid) + metadata = self.db.get_metadata(book_id) + title = metadata.get('title', 'Unknown') + + idx = list(self.sidecar_paths.keys()).index(book_uuid) + self.progress_update.emit(idx + 1, f'Calculating partial md5 hash: {title}') + + book_md5 = partial_md5_checksum(self.device, book_path) + + # Build metadata path + extension = re.sub(r'.*\.(\w+)$', r'\1', book_path) + sidecar_path = ( + f"{hashdocpath}/{book_md5[:2]}/{book_md5}.sdr/" + f"metadata.{extension}.lua" + ) + resolved_paths[book_uuid] = sidecar_path + debug_print(f'MD5 for {book_path}: {book_md5} -> {sidecar_path}') + + except Exception as e: + debug_print(f'Error calculating MD5 for {book_uuid}: {e}') + continue + else: + resolved_paths[book_uuid] = path_info + + self.sidecar_paths = resolved_paths + + for idx, (book_uuid, sidecar_path) in enumerate(self.sidecar_paths.items()): + debug_print('Trying to get sidecar from ', self.device, ', with sidecar_path: ', sidecar_path) # pre-checks before parsing if book_uuid is None: status = 'skipped, no UUID' append_results(results, None, status, - book_uuid, sidecar_path) + book_uuid, sidecar_path) num_skip += 1 continue sidecar_contents = self.action.get_sidecar( - device, sidecar_path) + self.device, sidecar_path) debug_print("sidecar_contents:", sidecar_contents) - book_id = db.lookup_by_uuid(book_uuid) - metadata = db.get_metadata(book_id) + book_id = self.db.lookup_by_uuid(book_uuid) + metadata = self.db.get_metadata(book_id) title = metadata.get('title') - self.progress_update.emit(idx + 1, title) - if DEBUG: # Add time delay when debugging + self.progress_update.emit(idx + 1, f'Processing: {title}') + if DEBUG: # Add time delay when debugging time.sleep(.4) if sidecar_contents is GetSidecarStatus.PATH_NOT_FOUND: status = ('skipped, sidecar does not exist ' - '(seems like book is never opened)') + '(seems like book is never opened)') append_results(results, title, status, - book_uuid, sidecar_path) + book_uuid, sidecar_path) num_skip += 1 continue elif sidecar_contents is GetSidecarStatus.DECODE_FAILED: status = 'decoding is failed see debug for more details' append_results(results, title, status, - book_uuid, sidecar_path) + book_uuid, sidecar_path) num_fail += 1 continue @@ -1181,7 +1343,7 @@ def run(self): keys_values_to_update[target] = value operation_status, result = self.action.update_metadata( - book_uuid, db, keys_values_to_update + book_uuid, self.db, keys_values_to_update ) results.append( @@ -1205,13 +1367,15 @@ def run(self): db = self.gui.current_db.new_api startTime = time.perf_counter() - self.koSyncWorker = KOSyncWorker(self, db, sidecar_paths) + self.koSyncWorker = KOSyncWorker(self, db, sidecar_paths, device) progress_dialog = None - if not silent and len(sidecar_paths) > 10: - progress_dialog = ProgressDialog( - self.gui, "Syncing Sidecars...", len(sidecar_paths)) - progress_dialog.show() - self.koSyncWorker.progress_update.connect(progress_dialog.setValue) + if not silent: + # Toujours montrer la progress bar si on utilise hashdocsettings + if CONFIG['checkbox_enable_sidecar_hashdocsettings'] or len(sidecar_paths) > 10: + progress_dialog = ProgressDialog( + self.gui, "Syncing Sidecars...", len(sidecar_paths)) + progress_dialog.show() + self.koSyncWorker.progress_update.connect(progress_dialog.setValue) def on_finished(res): if not silent: diff --git a/config.py b/config.py index cceda1b..e01287d 100644 --- a/config.py +++ b/config.py @@ -286,6 +286,11 @@ CONFIG.defaults[this_column] = '' for this_checkbox in CHECKBOXES: CONFIG.defaults[this_checkbox] = False +# hashdocsettings defautls +CONFIG.defaults['checkbox_enable_sidecar_hashdocsettings'] = False +CONFIG.defaults['sidecar_hashdocsettings_loc'] = '../koreader' +CONFIG.defaults['checkbox_autodetect_hashdocsettings'] = True +CONFIG.defaults['checkbox_save_detected_path'] = True CONFIG.defaults['progress_sync_url'] = 'https://sync.koreader.rocks:443' CONFIG.defaults['progress_sync_username'] = '' CONFIG.defaults['progress_sync_password'] = '' @@ -357,6 +362,98 @@ def __init__(self, plugin_action): layout.addLayout(self.add_checkbox('checkbox_enable_automatic_sync')) + # Hashdocsettings Section + layout.addWidget(create_separator()) + + hashdoc_header_label = QLabel( + "KOReader can store metadata sidecars in a centralized location using hashdocsettings. " + "Enable this option if you use hashdocsettings instead of storing sidecars alongside books. " + "You must specify the relative path to the KOReader directory from your books (e.g., ../koreader)." + ) + hashdoc_header_label.setWordWrap(True) + layout.addWidget(hashdoc_header_label) + + # Checkbox to enable hashdocsettings + hashdoc_checkbox_layout = QHBoxLayout() + hashdoc_checkbox = QCheckBox() + hashdoc_checkbox.setCheckState( + Qt.Checked if CONFIG['checkbox_enable_sidecar_hashdocsettings'] else Qt.Unchecked + ) + hashdoc_label = QLabel("Use hashdocsettings for metadata storage") + hashdoc_label.setToolTip( + "Enable this to use KOReader's hashdocsettings feature for centralized metadata storage.\n" + "This is useful if you have configured KOReader to store sidecars in a single directory." + ) + hashdoc_label.setBuddy(hashdoc_checkbox) + hashdoc_label.mousePressEvent = lambda event, cb=hashdoc_checkbox: cb.toggle() + hashdoc_checkbox_layout.addWidget(hashdoc_checkbox) + hashdoc_checkbox_layout.addWidget(hashdoc_label) + hashdoc_checkbox_layout.addStretch() + layout.addLayout(hashdoc_checkbox_layout) + + # Path input for hashdocsettings directory + hashdoc_path_layout = QHBoxLayout() + hashdoc_path_layout.setAlignment(Qt.AlignLeft) + hashdoc_path_label = QLabel("Hashdocsettings path:") + hashdoc_path_label.setToolTip( + "Path to the KOReader directory containing hashdocsettings metadata.\n" + "Examples:\n" + " - Kindle: /mnt/us/koreader\n" + " - Kobo: /mnt/onboard/.adds/koreader\n" + " - Custom device: adjust according to your setup" + ) + hashdoc_path_input = QLineEdit() + hashdoc_path_input.setText(CONFIG['sidecar_hashdocsettings_loc']) + hashdoc_path_input.setMinimumWidth(300) + hashdoc_path_input.setPlaceholderText("/mnt/us/koreader") + hashdoc_path_layout.addWidget(hashdoc_path_label) + hashdoc_path_layout.addWidget(hashdoc_path_input) + hashdoc_path_layout.addStretch() + layout.addLayout(hashdoc_path_layout) + + self.hashdoc_checkbox = hashdoc_checkbox + self.hashdoc_path_input = hashdoc_path_input + + # Auto-detect checkbox + autodetect_layout = QHBoxLayout() + autodetect_checkbox = QCheckBox() + autodetect_checkbox.setCheckState( + Qt.Checked if CONFIG.get('checkbox_autodetect_hashdocsettings', True) else Qt.Unchecked + ) + autodetect_label = QLabel("Auto-detect hashdocsettings path") + autodetect_label.setToolTip( + "Automatically find the hashdocsettings directory on the device.\n" + "Works for Kobo (.adds/koreader), Kindle (koreader), and other devices." + ) + autodetect_label.setBuddy(autodetect_checkbox) + autodetect_label.mousePressEvent = lambda event, cb=autodetect_checkbox: cb.toggle() + autodetect_layout.addWidget(autodetect_checkbox) + autodetect_layout.addWidget(autodetect_label) + autodetect_layout.addStretch() + layout.addLayout(autodetect_layout) + + # Save detected path checkbox + save_detected_layout = QHBoxLayout() + save_detected_checkbox = QCheckBox() + save_detected_checkbox.setCheckState( + Qt.Checked if CONFIG.get('checkbox_save_detected_path', True) else Qt.Unchecked + ) + save_detected_label = QLabel("Save auto-detected path for future syncs") + save_detected_label.setToolTip( + "Save the auto-detected path to configuration for faster subsequent syncs.\n" + "You can still manually override this path if needed." + ) + save_detected_label.setBuddy(save_detected_checkbox) + save_detected_label.mousePressEvent = lambda event, cb=save_detected_checkbox: cb.toggle() + save_detected_layout.addWidget(save_detected_checkbox) + save_detected_layout.addWidget(save_detected_label) + save_detected_layout.addStretch() + layout.addLayout(save_detected_layout) + + # Stocker les widgets pour les sauvegarder plus tard + self.autodetect_checkbox = autodetect_checkbox + self.save_detected_checkbox = save_detected_checkbox + # Progress Sync Section layout.addWidget(create_separator()) ps_header_label = QLabel( @@ -405,21 +502,43 @@ def save_settings(self): debug_print('old CONFIG = ', CONFIG) # Check relevant settings for changes in order to show restart warning - needRestart = (self.must_restart or # Custom Column Addition - CONFIG['checkbox_enable_automatic_sync'] != (CHECKBOXES['checkbox_enable_automatic_sync']['checkbox'].checkState() == Qt.Checked) or - CONFIG['checkbox_enable_scheduled_progressync'] != (CHECKBOXES['checkbox_enable_scheduled_progressync']['checkbox'].checkState() == Qt.Checked) or - CONFIG['scheduleSyncHour'] != self.schedule_hour_input.value() or - CONFIG['scheduleSyncMinute'] != self.schedule_minute_input.value() - ) + needRestart = ( + self.must_restart or # Custom Column Addition + CONFIG.get('checkbox_enable_automatic_sync', False) != ( + CHECKBOXES['checkbox_enable_automatic_sync']['checkbox'].checkState() == Qt.Checked + ) or + CONFIG.get('checkbox_enable_scheduled_progressync', False) != ( + CHECKBOXES['checkbox_enable_scheduled_progressync']['checkbox'].checkState() == Qt.Checked + ) or + CONFIG.get('scheduleSyncHour', 4) != self.schedule_hour_input.value() or + CONFIG.get('scheduleSyncMinute', 0) != self.schedule_minute_input.value() or + # Vérifier si hashdocsettings a changé + CONFIG.get('checkbox_enable_sidecar_hashdocsettings', False) != ( + self.hashdoc_checkbox.checkState() == Qt.Checked + ) or + CONFIG.get('sidecar_hashdocsettings_loc', '') != self.hashdoc_path_input.text().strip() + ) # Save Column Settings for config_name, metadata in CUSTOM_COLUMN_DEFAULTS.items(): CONFIG[config_name] = metadata['comboBox'].get_selected_column() - # Save Checkbox Settings + # Save Checkbox Settings (only those in CHECKBOXES dict with 'checkbox' attribute) for config_name in CHECKBOXES: - CONFIG[config_name] = CHECKBOXES[config_name]['checkbox'].checkState( - ) == Qt.Checked + if 'checkbox' in CHECKBOXES[config_name]: + CONFIG[config_name] = CHECKBOXES[config_name]['checkbox'].checkState() == Qt.Checked + + # Save Hashdocsettings Settings (gérés manuellement) + CONFIG['checkbox_enable_sidecar_hashdocsettings'] = ( + self.hashdoc_checkbox.checkState() == Qt.Checked + ) + CONFIG['sidecar_hashdocsettings_loc'] = self.hashdoc_path_input.text().strip() + CONFIG['checkbox_autodetect_hashdocsettings'] = ( + self.autodetect_checkbox.checkState() == Qt.Checked + ) + CONFIG['checkbox_save_detected_path'] = ( + self.save_detected_checkbox.checkState() == Qt.Checked + ) # Save Scheduled ProgressSync Settings CONFIG['scheduleSyncHour'] = self.schedule_hour_input.value() @@ -427,7 +546,10 @@ def save_settings(self): # NOTE: Server/Credentials are saved by the ProgressSyncPopup debug_print('new CONFIG = ', CONFIG) - if needRestart and show_restart_warning('Changes have been made that require a restart to take effect.\nRestart now?'): + + if needRestart and show_restart_warning( + 'Changes have been made that require a restart to take effect.\nRestart now?' + ): self.action.gui.quit(restart=True) def add_checkbox(self, checkboxKey): diff --git a/md5_utils.py b/md5_utils.py new file mode 100644 index 0000000..5f12dea --- /dev/null +++ b/md5_utils.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +import hashlib +import io +from typing import BinaryIO + +STEP = 1024 +SIZE = 1024 + + +def partial_md5_checksum(device, path: str) -> str: + """ + Compute the partial MD5 for file at `path` using the same algorithm + as KOReader, but retrieving the file via `device.get_file`. + + :param device: a device object providing get_file(path, outfile) + :param path: Path to the file on the device + :return: MD5 hex digest (lowercase) + :raises FileNotFoundError: If file doesn't exist or cannot be retrieved + """ + md5 = hashlib.md5() + + # Récupérer le fichier entier dans un buffer mémoire + with io.BytesIO() as outfile: + try: + device.get_file(path, outfile) + except Exception as e: + raise FileNotFoundError(f"Could not get file from device: {path}") from e + + contents = outfile.getvalue() + + # Vérifier que le fichier n'est pas vide + if not contents: + raise ValueError(f"File is empty: {path}") + + # On travaille maintenant sur contents comme si c'était le fichier + f = io.BytesIO(contents) # type: BinaryIO + file_size = len(contents) + + for i in range(-1, 11): # -1 .. 10 inclus + shift = (2 * i) & 31 + pos = (STEP << shift) & 0xFFFFFFFF + + # Ne pas essayer de lire au-delà de la taille du fichier + if pos >= file_size: + continue + + try: + f.seek(pos) + except (OSError, IOError): + continue + + chunk = f.read(SIZE) + + if chunk: + md5.update(chunk) + + return md5.hexdigest()