From 3374fcba9bb38d90ce8fd7c0ddda2f38ac9cd265 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 25 Sep 2026 14:27:48 +0000 Subject: [PATCH] filing: failed hardlink no longer writes the processed-book marker A cross-device (or otherwise failed) hardlink still cached the book under __cache__/book, which never expires, so the next run skipped it and the files never reached the library. Write the marker only when every file was filed or dedupe_roots already has the release. Co-authored-by: Mancolt --- CHANGELOG.md | 7 ++++ CONFIG.md | 5 ++- booktree.py | 29 +++++++++---- myx_classes.py | 10 ++++- tests/test_logmode.py | 51 +++++++++++++++++++++++ tests/test_operator_tools.py | 79 ++++++++++++++++++++++++++++++++++++ 6 files changed, 172 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4097689..9341e88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ first release is 3.0.0 because exit codes and the cookie store changed in ways a ## Unreleased +### Fixed +- A matched book whose hardlink (or copy) failed — typically `source_path` and `media_path` on different + filesystems (`EXDEV`) — was still given the never-expiring processed-book marker. Later runs printed + `Skipping: … already processed` and the files never reached the library. The marker is now written only + when every file was filed, or when `dedupe_roots` already has the release; a failed filing is retried + next run. + ## 3.0.4 - 2026-09-22 ### Fixed diff --git a/CONFIG.md b/CONFIG.md index 04a3090..613c715 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -191,7 +191,10 @@ the behaviour is unchanged: the first highest score wins. ## Correcting a match booktree processes a release once: after a match is filed, a processed marker is written for that release -(`__cache__/book/...`, no expiry) and later runs print `Skipping: ... already processed`. booktree never reads or +(`__cache__/book/...`, no expiry) and later runs print `Skipping: ... already processed`. A hardlink or copy +that fails (the usual case is `source_path` and `media_path` on different filesystems) is not a filing: the +marker is not written and the release is retried next run. Hardlinks require both paths on the same +filesystem; set `flags/hardlink` to `0` to copy instead. booktree never reads or writes Audiobookshelf's database; it only creates the folder, the hardlinks and a `metadata.opf` at filing time. So a wrong match is not re-applied on the next run, and a correction is never undone by booktree. diff --git a/booktree.py b/booktree.py index 2e8240a..fed4ed8 100644 --- a/booktree.py +++ b/booktree.py @@ -126,16 +126,18 @@ def buildTreeFromLog(files, logfile, cfg): else: print (f"Skipping {book[b].name}, already processed...") - # #Create Hardlinks + # #Create Hardlinks print (f"\nCreating Hardlinks for {len(matchedFiles)} matched books") for mb in matchedFiles: hardlinkUnlessFiled(mb, cfg) - #cache this book - unless it's a dry run - if (not dryRun): - mb.cacheMe("book", str(book[b]), cfg) + #the processed-book marker never expires: write it only when the book is actually in the library + if (not dryRun) and shouldWriteProcessedMarker(mb): + mb.cacheMe("book", str(book[b]), cfg) + elif not dryRun: + print(f"Not marking {mb.name} as processed: filing failed; it will be retried next run") - myx_utilities.printDivider() + myx_utilities.printDivider() #Logging processed files print (f"\nLogging {len(allFiles)} processed books") @@ -331,9 +333,11 @@ def buildTreeFromHybridSources(path, mediaPath, files, logfile, cfg): print (f"\nCreating Hardlinks for {len(matchedFiles)} matched books\n") for mb in matchedFiles: hardlinkUnlessFiled(mb, cfg) - #cache this book - unless it's a dry run - if (not dryRun): + #the processed-book marker never expires: write it only when the book is actually in the library + if (not dryRun) and shouldWriteProcessedMarker(mb): mb.cacheMe("book", str(book[b]), cfg) + elif not dryRun: + print(f"Not marking {mb.name} as processed: filing failed; it will be retried next run") myx_utilities.printDivider() @@ -417,6 +421,17 @@ def refreshRequested(hints, fullpath, root): return False +def shouldWriteProcessedMarker(mb): + """True when the never-expiring `__cache__/book` marker may be written. + + A failed hardlink (typical: `source_path` and `media_path` on different filesystems, EXDEV) used to + cache the book anyway; the next run skipped it and the files never reached the library. Deduped books + (`alreadyFiled`) are already in a library and should stay marked processed.""" + if getattr(mb, "alreadyFiled", None): + return True + return bool(mb.files) and all(f.isHardlinked for f in mb.files) + + def hardlinkUnlessFiled(mb, cfg): """Create the book's hardlinks (and OPF) unless Config/dedupe_roots says the same files are already filed in a library; then the book is only reported. A release the operator asked for (--refresh/--pin) is always filed: diff --git a/myx_classes.py b/myx_classes.py index f3fae2f..5d52c51 100644 --- a/myx_classes.py +++ b/myx_classes.py @@ -311,8 +311,11 @@ def hardlinkFile(self, source, target): self.isHardlinked=True except Exception as e: print (f"\tFailed due to {e}") + self.isHardlinked=False else: print (f"\tSkipped : {filename} exists") + #already at the destination (a previous run, or a sibling disc that linked before one failed) + self.isHardlinked=True return self.isHardlinked @@ -333,8 +336,10 @@ def copyFile(self, source, target): self.isHardlinked=True except Exception as e: print (f"\tFailed due to {e}") + self.isHardlinked=False else: print (f"\tSkipped : {filename} exists") + self.isHardlinked=True return self.isHardlinked @@ -1009,11 +1014,14 @@ def createHardLinks(self, cfg): else: #copy the file f.copyFile(f.fullPath, p) + #the processed marker means "in media_path"; a calibre ingest success must not hide a failed filing + mediaFiled = f.isHardlinked #setup Calibre Ingestion Path if ((add2calibre) and len(calibre_path) > 0): #hardlink the file - f.hardlinkFile(f.fullPath, calibre_path) + f.hardlinkFile(f.fullPath, calibre_path) + f.isHardlinked = mediaFiled #generate the OPF file print (f"\tGenerating OPF file ...") diff --git a/tests/test_logmode.py b/tests/test_logmode.py index 3923d8b..8c8ed05 100644 --- a/tests/test_logmode.py +++ b/tests/test_logmode.py @@ -152,6 +152,57 @@ def run_once(**over): self.assertEqual(len(booktree.httpx.calls) if hasattr(booktree.httpx, "calls") else 2, 2) +class FailedFilingNotCachedTest(unittest.TestCase): + """A cross-device (or otherwise failed) hardlink must not write the never-expiring processed marker.""" + + def test_exdev_is_retried_on_the_next_run(self): + import myx_classes + import myx_hints + with tempfile.TemporaryDirectory() as td: + src, media = os.path.join(td, "src"), os.path.join(td, "media") + os.makedirs(os.path.join(src, "junk")); os.makedirs(media) + source_file = os.path.join(src, "junk", "AudioTrack 01.mp3") + open(source_file, "wb").write(b"\x00" * 16) + fix = os.path.join(td, "fix.csv") + row = dict.fromkeys(myx_utilities.getLogHeaders().keys(), "") + row.update({"book": "junk", "file": source_file, "isMatched": "False", "isHardLinked": "False", "mamCount": "0", + "audibleMatchCount": "0", "metadatasource": "id3", "id3-matchRate": "0", "id3-asin": "B0CC3NZ34S", + "id3-title": "AudioTrack 01", "id3-authors": "unknown artist", "id3-duration": str(492 * 60), + "id3-language": "english", "sourcePath": src, "mediaPath": media}) + with open(fix, "w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=list(myx_utilities.getLogHeaders().keys())); w.writeheader(); w.writerow(row) + saved_httpx = booktree.httpx + saved_link = myx_classes.os.link + booktree.httpx = FakeAudible(by_asin={"B0CC3NZ34S": product("B0CC3NZ34S", "The Coworker", ["Freida McFadden"], 492)}) + + def run_once(): + myx_hints._cache.clear(); myx_hints._appliedRefresh.clear() + cfg = FakeConfig(td, **{"Config/metadata": "log", "Config/flags/verbose": 1}) + out = io.StringIO() + with contextlib.redirect_stdout(out): + booktree.buildTreeFromLog(fix, os.path.join(td, f"log{len(os.listdir(td))}.csv"), cfg) + return out.getvalue() + + try: + def boom(*_a, **_k): + raise OSError(18, "Invalid cross-device link") + myx_classes.os.link = boom + first = run_once() + dest = os.path.join(media, "Freida McFadden", "The Coworker", "AudioTrack 01.mp3") + self.assertFalse(os.path.exists(dest), first) + self.assertIn("Not marking junk as processed: filing failed; it will be retried next run", first) + self.assertNotIn("Skipping junk, already processed", first) + myx_classes.os.link = saved_link + second = run_once() + self.assertNotIn("Skipping junk, already processed", second) + self.assertIn("Pinned ASIN B0CC3NZ34S accepted", second) + self.assertTrue(os.path.exists(dest), second) + self.assertTrue(os.path.samefile(source_file, dest)) + finally: + myx_classes.os.link = saved_link + booktree.httpx = saved_httpx + + class SeriesRoundTripTest(unittest.TestCase): """upstream #27: the log writes seriesparts as "Name part" (no '#'), the reader split on '#' and lost the part.""" diff --git a/tests/test_operator_tools.py b/tests/test_operator_tools.py index 466b62d..aeb7fe3 100644 --- a/tests/test_operator_tools.py +++ b/tests/test_operator_tools.py @@ -490,5 +490,84 @@ def test_abs_validation_and_token_file(self): self.assertIsNone(myx_library.validateAbs(self.cfg(**{"Config/abs/url": "http://abs.example", "Config/abs/scan": 0}))) +class ProcessedMarkerTest(unittest.TestCase): + """The __cache__/book marker never expires; write it only when the book is actually in the library.""" + + def test_failed_hardlink_is_not_marked_processed_already_filed_is(self): + import booktree + import myx_classes + mb = myx_classes.MAMBook("Some Book") + self.assertFalse(booktree.shouldWriteProcessedMarker(mb)) # no files + bf = myx_classes.BookFile("book.m4b", "/dl/book.m4b", "/dl", "/lib") + mb.files.append(bf) + self.assertFalse(booktree.shouldWriteProcessedMarker(mb)) # hardlink/copy failed + bf.isHardlinked = True + self.assertTrue(booktree.shouldWriteProcessedMarker(mb)) + bf2 = myx_classes.BookFile("cd2.m4b", "/dl/cd2.m4b", "/dl", "/lib") + mb.files.append(bf2) + self.assertFalse(booktree.shouldWriteProcessedMarker(mb)) # one disc failed + mb.alreadyFiled = "/lib/Author/Some Book" + bf.isHardlinked = bf2.isHardlinked = False + self.assertTrue(booktree.shouldWriteProcessedMarker(mb)) # dedupe: already in a library + + def test_dest_exists_and_stale_flag_and_calibre_do_not_lie(self): + # dest already there counts as filed (a sibling disc linked last run); a stale isHardLinked=True + # from the log is cleared on EXDEV; a calibre ingest success must not hide a media-path failure. + import booktree + import myx_classes + with tempfile.TemporaryDirectory() as td: + src, media, calibre = os.path.join(td, "src"), os.path.join(td, "media"), os.path.join(td, "calibre") + os.makedirs(src); os.makedirs(media); os.makedirs(calibre) + source = os.path.join(src, "book.m4b") + open(source, "wb").write(b"\x00" * 16) + match = myx_classes.Book(asin=ASIN, title="Title") + match.authors = [myx_classes.Contributor("Lee Child")] + cfg = FakeConfig(td, **{"Config/flags/hardlink": 1, "Config/flags/no_opf": 1}) + + dest = os.path.join(media, "Lee Child", "Title", "book.m4b") + os.makedirs(os.path.dirname(dest)) + os.link(source, dest) + bf = myx_classes.BookFile("book.m4b", source, src, media) + mb = myx_classes.MAMBook("Title") + mb.files.append(bf) + mb.bestAudibleMatch = mb.metadataBook = match + mb.metadata = "audible" + with contextlib.redirect_stdout(io.StringIO()): + mb.createHardLinks(cfg) + self.assertTrue(bf.isHardlinked) + self.assertTrue(booktree.shouldWriteProcessedMarker(mb)) + + os.remove(dest) + bf.isHardlinked = True # as buildTreeFromLog would set from isHardLinked=True + saved = myx_classes.os.link + try: + myx_classes.os.link = lambda *_a, **_k: (_ for _ in ()).throw(OSError(18, "Invalid cross-device link")) + with contextlib.redirect_stdout(io.StringIO()): + mb.createHardLinks(cfg) + finally: + myx_classes.os.link = saved + self.assertFalse(bf.isHardlinked) + self.assertFalse(booktree.shouldWriteProcessedMarker(mb)) + + cfg = FakeConfig(td, **{"Config/flags/hardlink": 1, "Config/flags/no_opf": 1, + "Config/flags/ingest_calibre": 1, + "Config/target_path/calibre_ingest_path": calibre}) + real_link = os.link + def link_only_calibre(src_path, dst_path): + if dst_path.startswith(calibre): + return real_link(src_path, dst_path) + raise OSError(18, "Invalid cross-device link") + try: + myx_classes.os.link = link_only_calibre + with contextlib.redirect_stdout(io.StringIO()): + mb.createHardLinks(cfg) + finally: + myx_classes.os.link = saved + self.assertTrue(os.path.exists(os.path.join(calibre, "book.m4b"))) + self.assertFalse(os.path.exists(dest)) + self.assertFalse(bf.isHardlinked) + self.assertFalse(booktree.shouldWriteProcessedMarker(mb)) + + if __name__ == "__main__": unittest.main()