filing: failed hardlink no longer writes the processed-book marker

CHANGELOG.md

@@ -6,6 +6,13 @@ first release is 3.0.0 because exit codes and the cookie store changed in ways a
 
 ## Unreleased
 
+### Fixed
+- A matched book whose hardlink (or copy) failed — typically `source_path` and `media_path` on different
+  filesystems (`EXDEV`) — was still given the never-expiring processed-book marker. Later runs printed
+  `Skipping: … already processed` and the files never reached the library. The marker is now written only
+  when every file was filed, or when `dedupe_roots` already has the release; a failed filing is retried
+  next run.
+
 ## 3.0.4 - 2026-09-22
 
 ### Fixed

CONFIG.md

@@ -191,7 +191,10 @@ the behaviour is unchanged: the first highest score wins.
 ## Correcting a match
 
 booktree processes a release once: after a match is filed, a processed marker is written for that release
-(`__cache__/book/...`, no expiry) and later runs print `Skipping: ... already processed`. booktree never reads or
+(`__cache__/book/...`, no expiry) and later runs print `Skipping: ... already processed`. A hardlink or copy
+that fails (the usual case is `source_path` and `media_path` on different filesystems) is not a filing: the
+marker is not written and the release is retried next run. Hardlinks require both paths on the same
+filesystem; set `flags/hardlink` to `0` to copy instead. booktree never reads or
 writes Audiobookshelf's database; it only creates the folder, the hardlinks and a `metadata.opf` at filing time.
 So a wrong match is not re-applied on the next run, and a correction is never undone by booktree.
 

booktree.py

@@ -126,16 +126,18 @@ def buildTreeFromLog(files, logfile, cfg):
             else:
                 print (f"Skipping {book[b].name}, already processed...")
 
-        # #Create Hardlinks
+        #         #Create Hardlinks
         print (f"\nCreating Hardlinks for {len(matchedFiles)} matched books")
         for mb in matchedFiles:
             hardlinkUnlessFiled(mb, cfg)
 
-            #cache this book - unless it's a dry run
-            if (not dryRun):
-                mb.cacheMe("book", str(book[b]), cfg)       
+            #the processed-book marker never expires: write it only when the book is actually in the library
+            if (not dryRun) and shouldWriteProcessedMarker(mb):
+                mb.cacheMe("book", str(book[b]), cfg)
+            elif not dryRun:
+                print(f"Not marking {mb.name} as processed: filing failed; it will be retried next run")
 
-            myx_utilities.printDivider()      
+            myx_utilities.printDivider()
 
         #Logging processed files
         print (f"\nLogging {len(allFiles)} processed books")
@@ -331,9 +333,11 @@ def buildTreeFromHybridSources(path, mediaPath, files, logfile, cfg):
     print (f"\nCreating Hardlinks for {len(matchedFiles)} matched books\n")
     for mb in matchedFiles:
         hardlinkUnlessFiled(mb, cfg)
-        #cache this book - unless it's a dry run
-        if (not dryRun):
+        #the processed-book marker never expires: write it only when the book is actually in the library
+        if (not dryRun) and shouldWriteProcessedMarker(mb):
             mb.cacheMe("book", str(book[b]), cfg)
+        elif not dryRun:
+            print(f"Not marking {mb.name} as processed: filing failed; it will be retried next run")
 
         myx_utilities.printDivider()
 
@@ -417,6 +421,17 @@ def refreshRequested(hints, fullpath, root):
     return False
 
 
+def shouldWriteProcessedMarker(mb):
+    """True when the never-expiring `__cache__/book` marker may be written.
+
+    A failed hardlink (typical: `source_path` and `media_path` on different filesystems, EXDEV) used to
+    cache the book anyway; the next run skipped it and the files never reached the library. Deduped books
+    (`alreadyFiled`) are already in a library and should stay marked processed."""
+    if getattr(mb, "alreadyFiled", None):
+        return True
+    return bool(mb.files) and all(f.isHardlinked for f in mb.files)
+
+
 def hardlinkUnlessFiled(mb, cfg):
     """Create the book's hardlinks (and OPF) unless Config/dedupe_roots says the same files are already filed in a
     library; then the book is only reported. A release the operator asked for (--refresh/--pin) is always filed:

myx_classes.py

@@ -311,8 +311,11 @@ def hardlinkFile(self, source, target):
                 self.isHardlinked=True
             except Exception as e:
                 print (f"\tFailed due to {e}")
+                self.isHardlinked=False
         else:
             print (f"\tSkipped : {filename} exists")
+            #already at the destination (a previous run, or a sibling disc that linked before one failed)
+            self.isHardlinked=True
 
         return self.isHardlinked
 
@@ -333,8 +336,10 @@ def copyFile(self, source, target):
                 self.isHardlinked=True
             except Exception as e:
                 print (f"\tFailed due to {e}")
+                self.isHardlinked=False
         else:
             print (f"\tSkipped : {filename} exists")
+            self.isHardlinked=True
 
         return self.isHardlinked
 
@@ -1009,11 +1014,14 @@ def createHardLinks(self, cfg):
                     else:
                         #copy the file
                         f.copyFile(f.fullPath, p)
+                    #the processed marker means "in media_path"; a calibre ingest success must not hide a failed filing
+                    mediaFiled = f.isHardlinked
 
                     #setup Calibre Ingestion Path
                     if ((add2calibre) and len(calibre_path) > 0):
                         #hardlink the file
-                        f.hardlinkFile(f.fullPath, calibre_path)                   
+                        f.hardlinkFile(f.fullPath, calibre_path)
+                        f.isHardlinked = mediaFiled
 
                     #generate the OPF file
                     print (f"\tGenerating OPF file ...")

tests/test_logmode.py

@@ -152,6 +152,57 @@ def run_once(**over):
         self.assertEqual(len(booktree.httpx.calls) if hasattr(booktree.httpx, "calls") else 2, 2)
 
 
+class FailedFilingNotCachedTest(unittest.TestCase):
+    """A cross-device (or otherwise failed) hardlink must not write the never-expiring processed marker."""
+
+    def test_exdev_is_retried_on_the_next_run(self):
+        import myx_classes
+        import myx_hints
+        with tempfile.TemporaryDirectory() as td:
+            src, media = os.path.join(td, "src"), os.path.join(td, "media")
+            os.makedirs(os.path.join(src, "junk")); os.makedirs(media)
+            source_file = os.path.join(src, "junk", "AudioTrack 01.mp3")
+            open(source_file, "wb").write(b"\x00" * 16)
+            fix = os.path.join(td, "fix.csv")
+            row = dict.fromkeys(myx_utilities.getLogHeaders().keys(), "")
+            row.update({"book": "junk", "file": source_file, "isMatched": "False", "isHardLinked": "False", "mamCount": "0",
+                        "audibleMatchCount": "0", "metadatasource": "id3", "id3-matchRate": "0", "id3-asin": "B0CC3NZ34S",
+                        "id3-title": "AudioTrack 01", "id3-authors": "unknown artist", "id3-duration": str(492 * 60),
+                        "id3-language": "english", "sourcePath": src, "mediaPath": media})
+            with open(fix, "w", newline="", encoding="utf-8") as fh:
+                w = csv.DictWriter(fh, fieldnames=list(myx_utilities.getLogHeaders().keys())); w.writeheader(); w.writerow(row)
+            saved_httpx = booktree.httpx
+            saved_link = myx_classes.os.link
+            booktree.httpx = FakeAudible(by_asin={"B0CC3NZ34S": product("B0CC3NZ34S", "The Coworker", ["Freida McFadden"], 492)})
+
+            def run_once():
+                myx_hints._cache.clear(); myx_hints._appliedRefresh.clear()
+                cfg = FakeConfig(td, **{"Config/metadata": "log", "Config/flags/verbose": 1})
+                out = io.StringIO()
+                with contextlib.redirect_stdout(out):
+                    booktree.buildTreeFromLog(fix, os.path.join(td, f"log{len(os.listdir(td))}.csv"), cfg)
+                return out.getvalue()
+
+            try:
+                def boom(*_a, **_k):
+                    raise OSError(18, "Invalid cross-device link")
+                myx_classes.os.link = boom
+                first = run_once()
+                dest = os.path.join(media, "Freida McFadden", "The Coworker", "AudioTrack 01.mp3")
+                self.assertFalse(os.path.exists(dest), first)
+                self.assertIn("Not marking junk as processed: filing failed; it will be retried next run", first)
+                self.assertNotIn("Skipping junk, already processed", first)
+                myx_classes.os.link = saved_link
+                second = run_once()
+                self.assertNotIn("Skipping junk, already processed", second)
+                self.assertIn("Pinned ASIN B0CC3NZ34S accepted", second)
+                self.assertTrue(os.path.exists(dest), second)
+                self.assertTrue(os.path.samefile(source_file, dest))
+            finally:
+                myx_classes.os.link = saved_link
+                booktree.httpx = saved_httpx
+
+
 class SeriesRoundTripTest(unittest.TestCase):
     """upstream #27: the log writes seriesparts as "Name part" (no '#'), the reader split on '#' and lost the part."""
 

tests/test_operator_tools.py

@@ -490,5 +490,84 @@ def test_abs_validation_and_token_file(self):
         self.assertIsNone(myx_library.validateAbs(self.cfg(**{"Config/abs/url": "http://abs.example", "Config/abs/scan": 0})))
 
 
+class ProcessedMarkerTest(unittest.TestCase):
+    """The __cache__/book marker never expires; write it only when the book is actually in the library."""
+
+    def test_failed_hardlink_is_not_marked_processed_already_filed_is(self):
+        import booktree
+        import myx_classes
+        mb = myx_classes.MAMBook("Some Book")
+        self.assertFalse(booktree.shouldWriteProcessedMarker(mb))          # no files
+        bf = myx_classes.BookFile("book.m4b", "/dl/book.m4b", "/dl", "/lib")
+        mb.files.append(bf)
+        self.assertFalse(booktree.shouldWriteProcessedMarker(mb))          # hardlink/copy failed
+        bf.isHardlinked = True
+        self.assertTrue(booktree.shouldWriteProcessedMarker(mb))
+        bf2 = myx_classes.BookFile("cd2.m4b", "/dl/cd2.m4b", "/dl", "/lib")
+        mb.files.append(bf2)
+        self.assertFalse(booktree.shouldWriteProcessedMarker(mb))          # one disc failed
+        mb.alreadyFiled = "/lib/Author/Some Book"
+        bf.isHardlinked = bf2.isHardlinked = False
+        self.assertTrue(booktree.shouldWriteProcessedMarker(mb))           # dedupe: already in a library
+
+    def test_dest_exists_and_stale_flag_and_calibre_do_not_lie(self):
+        # dest already there counts as filed (a sibling disc linked last run); a stale isHardLinked=True
+        # from the log is cleared on EXDEV; a calibre ingest success must not hide a media-path failure.
+        import booktree
+        import myx_classes
+        with tempfile.TemporaryDirectory() as td:
+            src, media, calibre = os.path.join(td, "src"), os.path.join(td, "media"), os.path.join(td, "calibre")
+            os.makedirs(src); os.makedirs(media); os.makedirs(calibre)
+            source = os.path.join(src, "book.m4b")
+            open(source, "wb").write(b"\x00" * 16)
+            match = myx_classes.Book(asin=ASIN, title="Title")
+            match.authors = [myx_classes.Contributor("Lee Child")]
+            cfg = FakeConfig(td, **{"Config/flags/hardlink": 1, "Config/flags/no_opf": 1})
+
+            dest = os.path.join(media, "Lee Child", "Title", "book.m4b")
+            os.makedirs(os.path.dirname(dest))
+            os.link(source, dest)
+            bf = myx_classes.BookFile("book.m4b", source, src, media)
+            mb = myx_classes.MAMBook("Title")
+            mb.files.append(bf)
+            mb.bestAudibleMatch = mb.metadataBook = match
+            mb.metadata = "audible"
+            with contextlib.redirect_stdout(io.StringIO()):
+                mb.createHardLinks(cfg)
+            self.assertTrue(bf.isHardlinked)
+            self.assertTrue(booktree.shouldWriteProcessedMarker(mb))
+
+            os.remove(dest)
+            bf.isHardlinked = True                    # as buildTreeFromLog would set from isHardLinked=True
+            saved = myx_classes.os.link
+            try:
+                myx_classes.os.link = lambda *_a, **_k: (_ for _ in ()).throw(OSError(18, "Invalid cross-device link"))
+                with contextlib.redirect_stdout(io.StringIO()):
+                    mb.createHardLinks(cfg)
+            finally:
+                myx_classes.os.link = saved
+            self.assertFalse(bf.isHardlinked)
+            self.assertFalse(booktree.shouldWriteProcessedMarker(mb))
+
+            cfg = FakeConfig(td, **{"Config/flags/hardlink": 1, "Config/flags/no_opf": 1,
+                                    "Config/flags/ingest_calibre": 1,
+                                    "Config/target_path/calibre_ingest_path": calibre})
+            real_link = os.link
+            def link_only_calibre(src_path, dst_path):
+                if dst_path.startswith(calibre):
+                    return real_link(src_path, dst_path)
+                raise OSError(18, "Invalid cross-device link")
+            try:
+                myx_classes.os.link = link_only_calibre
+                with contextlib.redirect_stdout(io.StringIO()):
+                    mb.createHardLinks(cfg)
+            finally:
+                myx_classes.os.link = saved
+            self.assertTrue(os.path.exists(os.path.join(calibre, "book.m4b")))
+            self.assertFalse(os.path.exists(dest))
+            self.assertFalse(bf.isHardlinked)
+            self.assertFalse(booktree.shouldWriteProcessedMarker(mb))
+
+
 if __name__ == "__main__":
     unittest.main()