diff --git a/README.md b/README.md index 8e0142849..3f4811929 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ Clean-room Go rewrite, modern React UI, MIT-licensed, actively developed. - Dual-format books — each title holds an ebook *and* an audiobook in independent slots, with separate search, grab, and import pipelines, and the audiobook side moves multi-part `.m4b` / `.mp3` folders as one unit. - Series support with position tracking and search by series title, edition tracking (format / ISBN / publisher / page count), Calendar view of upcoming releases, and multiple library roots. - Library scan with four-tier matching: ASIN → title + author → series name + position → fuzzy title. Honours librarian sort-suffix form (`Title, The`) and series-annotated filenames (`[Mistborn, Book 1]`). +- **Library adoption**: whatever the scan could not match waits on the Import page as one row per book (an audiobook folder or disc set is one row), with suggested matches (one click Confirm only for a strong match), one "Add author" decision per missing author, and Undo. Files are registered where they are, never moved. - Author aliases (`RR Haywood` / `R.R. Haywood` / `R R Haywood` merge into one canonical row), and metadata re-bind to correct a wrong match without delete-and-re-add. On a book’s **File → Re-bind** dialog, search configured metadata providers and select an OpenLibrary work or Hardcover book, or enter an exact provider ID and press Enter. Results already in the library are marked and link to the existing book; records belonging to another library book cannot be selected. Each result’s **Links** menu opens its upstream page when available; OpenLibrary editions can be viewed but require a work ID for re-binding. Re-binding updates metadata and series without moving files; author mismatches require confirmation. - Explicit author-catalogue reconciliation with a selectable preview: remove chosen stale metadata-only Wanted rows after changing provider or metadata profile while always protecting imported books and every row with a tracked file. diff --git a/changelog.d/library-adoption.md b/changelog.d/library-adoption.md new file mode 100644 index 000000000..20345183a --- /dev/null +++ b/changelog.d/library-adoption.md @@ -0,0 +1,5 @@ +### Added +- **Adopt the books a library scan could not match** (#2547): they now wait on the Import page under **In your library**, which is where `/import` opens. Each row is a book rather than a file, so a 193 track audiobook is one row, a disc set is one row named after its folder, and `Dune.epub` beside `Dune.mobi` is one row. A close title by the same author is one click to confirm, a weaker one is offered as a possible match to check first. Books by an author who is not in your library yet are one row with a single Add author. Any row opens in place to pick a book, search your library, or search metadata, and metadata providers are only asked when you press Search. Adopting registers the files where they are: nothing is moved and nothing is searched for, a book added from metadata comes in unmonitored in the format you adopted, and Undo reverses it exactly. Ignored books stay hidden across scans until you unignore them. Admins see how many books need a decision on the Import nav entry. + +### Changed +- **The unmatched files table in Settings and the bulk folder scan in Settings > Import moved to the Import page.** Settings now shows how many books need a decision with a link to review them, and the folder scan is **Import > From a folder**. The old list stopped at 1000 files; the new one has no such cap for ordinary libraries (it lists up to 20,000 books per scan and says when there are more). The `library.lastScan` summary keeps its counters, adds `unmatched_units`, `ignored_units` and `units_truncated`, and returns `unmatched_files` as an empty list for one release. diff --git a/cmd/bindery/adoption_routes_test.go b/cmd/bindery/adoption_routes_test.go new file mode 100644 index 000000000..e62e018bb --- /dev/null +++ b/cmd/bindery/adoption_routes_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "sort" + "strings" + "testing" + + "github.com/go-chi/chi/v5" + + "github.com/vavallee/bindery/internal/auth" +) + +// stubAdoptionHandler records which adoption route ran. +type stubAdoptionHandler struct{ called []string } + +func (h *stubAdoptionHandler) record(name string, w http.ResponseWriter) { + h.called = append(h.called, name) + w.WriteHeader(http.StatusNoContent) +} + +func (h *stubAdoptionHandler) List(w http.ResponseWriter, _ *http.Request) { h.record("list", w) } +func (h *stubAdoptionHandler) Summary(w http.ResponseWriter, _ *http.Request) { h.record("summary", w) } +func (h *stubAdoptionHandler) Adopt(w http.ResponseWriter, _ *http.Request) { h.record("adopt", w) } +func (h *stubAdoptionHandler) Undo(w http.ResponseWriter, _ *http.Request) { h.record("undo", w) } +func (h *stubAdoptionHandler) Ignore(w http.ResponseWriter, _ *http.Request) { h.record("ignore", w) } +func (h *stubAdoptionHandler) Unignore(w http.ResponseWriter, _ *http.Request) { + h.record("unignore", w) +} +func (h *stubAdoptionHandler) IgnoreMany(w http.ResponseWriter, _ *http.Request) { + h.record("ignore-many", w) +} + +// adoptionRoutes is every route registerAdoptionRoutes mounts, with a concrete +// path for each and the handler it must reach. +var adoptionRoutes = []struct { + method, pattern, path, called string +}{ + {http.MethodGet, "/library/unmatched", "/library/unmatched", "list"}, + {http.MethodGet, "/library/unmatched/summary", "/library/unmatched/summary", "summary"}, + {http.MethodPost, "/library/unmatched/ignore", "/library/unmatched/ignore", "ignore-many"}, + {http.MethodPost, "/library/unmatched/{id}/adopt", "/library/unmatched/7/adopt", "adopt"}, + {http.MethodPost, "/library/unmatched/{id}/undo", "/library/unmatched/7/undo", "undo"}, + {http.MethodPost, "/library/unmatched/{id}/ignore", "/library/unmatched/7/ignore", "ignore"}, + {http.MethodPost, "/library/unmatched/{id}/unignore", "/library/unmatched/7/unignore", "unignore"}, +} + +// TestAdoptionRoutesEnumerated walks the mounted router so a route added to +// registerAdoptionRoutes without a row in adoptionRoutes (and so without the +// admin checks below) fails here. +func TestAdoptionRoutesEnumerated(t *testing.T) { + router := chi.NewRouter() + registerAdoptionRoutes(router, &stubAdoptionHandler{}) + var mounted, want []string + if err := chi.Walk(router, func(method, route string, _ http.Handler, _ ...func(http.Handler) http.Handler) error { + mounted = append(mounted, method+" "+route) + return nil + }); err != nil { + t.Fatal(err) + } + for _, r := range adoptionRoutes { + want = append(want, r.method+" "+r.pattern) + } + sort.Strings(mounted) + sort.Strings(want) + if strings.Join(mounted, "\n") != strings.Join(want, "\n") { + t.Fatalf("mounted adoption routes:\n%s\nwant:\n%s", strings.Join(mounted, "\n"), strings.Join(want, "\n")) + } +} + +// TestAdoptionRoutesRequireAdmin: the rows carry absolute library paths and +// adopt writes authors, books and book files, so a user role, or no role at +// all, is refused before any handler runs. +func TestAdoptionRoutesRequireAdmin(t *testing.T) { + for _, role := range []string{"user", ""} { + for _, rt := range adoptionRoutes { + t.Run(role+" "+rt.method+" "+rt.path, func(t *testing.T) { + h := &stubAdoptionHandler{} + router := chi.NewRouter() + registerAdoptionRoutes(router, h) + req := httptest.NewRequest(rt.method, rt.path, nil) + if role != "" { + req = req.WithContext(auth.WithUserRole(req.Context(), role)) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden || len(h.called) != 0 { + t.Fatalf("status = %d, called %v; want 403 and no handler", rec.Code, h.called) + } + }) + } + } +} + +// TestAdoptionRoutesAllowAdmin is the symmetry case. +func TestAdoptionRoutesAllowAdmin(t *testing.T) { + for _, rt := range adoptionRoutes { + t.Run(rt.method+" "+rt.path, func(t *testing.T) { + h := &stubAdoptionHandler{} + router := chi.NewRouter() + registerAdoptionRoutes(router, h) + req := httptest.NewRequest(rt.method, rt.path, nil) + req = req.WithContext(auth.WithUserRole(req.Context(), "admin")) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusNoContent || len(h.called) != 1 || h.called[0] != rt.called { + t.Fatalf("status = %d, called %v; want 204 and %s", rec.Code, h.called, rt.called) + } + }) + } +} diff --git a/cmd/bindery/main.go b/cmd/bindery/main.go index f5a2848aa..92f03c617 100644 --- a/cmd/bindery/main.go +++ b/cmd/bindery/main.go @@ -701,6 +701,18 @@ func main() { } libraryHandler := api.NewLibraryHandler(importScanner).WithSettings(settingsRepo) + // Library adoption: the scan stores the books it could not match, and the + // Import page's "In your library" view acts on them. Library roots, not + // importRoots: adoption only registers files already in the library. + unmatchedUnitRepo := db.NewUnmatchedUnitRepo(database) + importScanner.WithUnmatchedUnits(unmatchedUnitRepo) + adoptionHandler := api.NewAdoptionHandler(unmatchedUnitRepo, bookRepo, authorRepo, authorHandler, libraryRoots, importScanner, settingsRepo) + // Any claim present at startup belongs to a process that is gone. + if n, err := adoptionHandler.RecoverStaleClaims(ctxBoot, 0); err != nil { + slog.Warn("library adoption: could not recover abandoned claims", "error", err) + } else if n > 0 { + slog.Info("library adoption: recovered abandoned claims", "count", n) + } fileHandler := api.NewFileHandler(bookRepo, cfg.LibraryDir, cfg.AudiobookDir). WithRootFolders(rootFolderRepo) historyHandler := api.NewHistoryHandler(historyRepo, blocklistRepo, bookRepo) @@ -1138,6 +1150,8 @@ func main() { // admin only (#2361); see registerLibraryScanStatusRoute. r.Post("/library/scan", libraryHandler.Scan) registerLibraryScanStatusRoute(r, libraryHandler) + // Library adoption, admin only (see registerAdoptionRoutes). + registerAdoptionRoutes(r, adoptionHandler) // Refresh metadata for ALL authors (background job, #863). Per-selection // bulk refresh lives at /author/bulk; this is the "populate everything" diff --git a/cmd/bindery/sensitive_routes.go b/cmd/bindery/sensitive_routes.go index e0e707456..e2794e887 100644 --- a/cmd/bindery/sensitive_routes.go +++ b/cmd/bindery/sensitive_routes.go @@ -201,6 +201,35 @@ func registerStorageRoutes(r chi.Router, h storageRouteHandler) { }) } +// adoptionRouteHandler is the surface registerAdoptionRoutes needs. +type adoptionRouteHandler interface { + List(http.ResponseWriter, *http.Request) + Summary(http.ResponseWriter, *http.Request) + Adopt(http.ResponseWriter, *http.Request) + Undo(http.ResponseWriter, *http.Request) + Ignore(http.ResponseWriter, *http.Request) + Unignore(http.ResponseWriter, *http.Request) + IgnoreMany(http.ResponseWriter, *http.Request) +} + +// registerAdoptionRoutes mounts /library/unmatched, library adoption, admin +// only. The rows carry absolute library paths (the same disclosure +// registerLibraryScanStatusRoute gates), adopt creates authors and books and +// writes book_files, and the summary behind the nav badge counts them. No +// route takes a path: every row is addressed by id. +func registerAdoptionRoutes(r chi.Router, h adoptionRouteHandler) { + r.Group(func(r chi.Router) { + r.Use(auth.RequireAdmin) + r.Get("/library/unmatched", h.List) + r.Get("/library/unmatched/summary", h.Summary) + r.Post("/library/unmatched/ignore", h.IgnoreMany) + r.Post("/library/unmatched/{id}/adopt", h.Adopt) + r.Post("/library/unmatched/{id}/undo", h.Undo) + r.Post("/library/unmatched/{id}/ignore", h.Ignore) + r.Post("/library/unmatched/{id}/unignore", h.Unignore) + }) +} + // libraryScanStatusRouteHandler is the surface registerLibraryScanStatusRoute // needs. type libraryScanStatusRouteHandler interface { diff --git a/docs/API.md b/docs/API.md index 515c3c7f1..958fb7f6e 100644 --- a/docs/API.md +++ b/docs/API.md @@ -374,6 +374,13 @@ POST /api/v1/backup/{filename}/restore stage a backup for the next re GET /api/v1/system/status version, uptime, build info POST /api/v1/library/scan start a library scan in the background (202) GET /api/v1/library/scan/status summary of the last library scan, paths included (admin) +GET /api/v1/library/unmatched books the scan could not match, one row per book (admin) +GET /api/v1/library/unmatched/summary pending, ignored and adopted counts plus scan status (admin) +POST /api/v1/library/unmatched/{id}/adopt register the row's files in place against a book (admin) +POST /api/v1/library/unmatched/{id}/undo reverse an adoption exactly (admin) +POST /api/v1/library/unmatched/{id}/ignore set a pending row aside (admin) +POST /api/v1/library/unmatched/{id}/unignore return an ignored row to pending (admin) +POST /api/v1/library/unmatched/ignore ignore pending rows by {"ids":[..]} or {"authorFolder":"..."} (admin) PUT /api/v1/system/loglevel runtime log-level switch (debug/info/warn/error) GET /api/v1/images?url= proxied + cached cover image (30-day TTL) ``` @@ -382,7 +389,68 @@ GET /api/v1/images?url= proxied + cached cover image ( scan: the counts, the library roots it walked and the path of every unmatched file. That is server filesystem layout, so the route is admin only in the same way as `/system/storage`, and a non admin gets `403` (#2361). A `404` means no -scan has run yet. +scan has run yet. The unmatched files themselves moved to +`/library/unmatched`: the summary now carries `unmatched_units`, +`ignored_units` and `units_truncated`, and `unmatched_files` is always an +empty list, kept for one release so an older cached web bundle still parses. + +#### Library adoption + +`GET /api/v1/library/unmatched` lists the books a library scan could not +match. A row is a book: an audiobook folder, a disc set, or same named ebook +files are one row. Query parameters, all optional: + +| Parameter | Values | +|---|---| +| `state` | `pending` (default), `ignored`, `adopted` | +| `reason` | `author_not_in_library`, `no_candidate_books`, `no_title_match`, `no_title_parsed` | +| `authorFolder` | first folder under the library root | +| `format` | `ebook`, `audiobook` | +| `search` | words matched against title, author and path | +| `sort`, `dir` | `score` (default), `title`, `folder`, `files`, `size`, `seen`; `asc` or `desc` | +| `limit`, `offset` | page, at most 250 rows | +| `facets` | `1` adds grouped counts by reason, format and top author folders | + +The response is `{items, total, facets?, summary, scan}`. Each item carries +`id`, `kind`, `format`, `fileCount`, `sizeBytes`, `relPath`, `rootPath`, +`authorFolder`, `parsedTitle`, `parsedAuthor`, `reason`, up to three +`candidates` (`{book, score}`, a title similarity from 0 to 1), `state`, the +adopted `book` if any, `bookCreated`, `authorCreated` and the first 20 +`members` file names. No provider is called to build it. + +`POST /api/v1/library/unmatched/{id}/adopt` takes either `{"bookId": 12}` for +a book already in the library or `{"foreignBookId": "...", "foreignAuthorId": +"...", "authorName": "..."}` for a metadata result. An optional `"format"` +must equal the row's format; audio files cannot be adopted as an ebook. No +field is a path. The files are registered in place (an audiobook folder as its +folder, a disc set disc folder by disc folder), nothing is moved and no search +starts. A book added from metadata is created unmonitored with its media type +set to the files' format. Every side effect is recorded on the row before the +next, so an adoption interrupted by a crash is reversed at the next start. +Answers: + +| Status | Meaning | +|---|---| +| `200` | the updated row | +| `400` | neither or both of `bookId` and `foreignBookId`, or a format that is not the files' format | +| `404` | no such row, or the book is gone | +| `409` | the row is not pending (the body names its `state`), or a file already belongs to a book | +| `422` | a file is gone, is not a regular file, or resolves outside the library folders | +| `502`, `503` | the metadata provider failed or the primary provider is down | + +`POST .../undo` removes exactly the book file entries the adoption made, each +only while it still belongs to the book it was registered to, and a book or +author it created when nothing else holds them (books by the author count +whether excluded or not) and the book has not been used since (monitored, +edited, linked to a series, searched or downloaded). A kept book is reported in +the response's `message`. The row returns to pending only once every step +succeeded; a `503` means a step failed (a busy database, say), the adoption +is still on record and is completed automatically within a few minutes, and +a `409` means another request took the row before the undo finished. An adopt +that fails and cannot reverse what it did also answers `503` and is cleaned up +the same way. Adopt, undo, ignore and unignore each claim the row with a compare and +swap, so a double click or two admins acting at once get one `200` and one +`409`. #### Webhook payload diff --git a/docs/Troubleshooting-Wiki.md b/docs/Troubleshooting-Wiki.md index ebd10467a..34c9fd192 100644 --- a/docs/Troubleshooting-Wiki.md +++ b/docs/Troubleshooting-Wiki.md @@ -313,8 +313,8 @@ co-author or a joint pen name. The log shows lines like: DEBUG calibre import: alias record skipped error="alias \"Isaac Asimov\" already points at author 1125 (refusing to reassign to 105)" name="Isaac Asimov" ``` -The scan then reports those same names under **Unmatched files** with "Parsed -author isn't in your library", because the name only exists as an alias. +The scan then lists those books on **Import → In your library** as by an +author who is not in your library, because the name only exists as an alias. What happened: for a book credited to several people, the import used to record every co-author as an *alias* of the first credited author ([#1684](https://github.com/vavallee/bindery/issues/1684)). @@ -374,6 +374,29 @@ Two things this does not do: If you reshape your library regularly, **Rename files** on the book or author page is the supported way to do it: it moves the file *and* repoints the same tracking row at the new location, so there is never a second row to clean up. +## A scan leaves files unmatched + +Go to **Import → In your library**. Every book the last scan could not match +is listed there, one row per book, with a sentence saying what the scan found +and what to do: add the missing author and scan again, confirm a suggested +book, or choose one. Adopting registers the files where they are and can be +undone. The full walkthrough is in the user guide under [Adopting files +already in your library](User-Guide-Wiki.md#adopting-files-already-in-your-library). + +A few things that look wrong but are not: + +- **A file you expected is not listed.** Symlinks are never listed, and a file + whose folder resolves outside your library folders is skipped. Replace the + link with the file, or add the real folder as a root. +- **The list did not change after the volume came back.** A scan that finds no + files at all changes nothing on the list, so an unmounted library does not + erase your decisions. Scan again once it is mounted. +- **A row you ignored is gone for good.** It is on the **Ignored** list, and + **Unignore** brings it back. + +If you file a bug about a row, include the reason code that shows when you +hover its sentence. + ## Collecting logs for a bug report `Settings → Logs` is the whole log store, so you don't need shell access to the container (rootless images give you nowhere to `cat` a file anyway). diff --git a/docs/User-Guide-Wiki.md b/docs/User-Guide-Wiki.md index f4ec4308c..ba8815598 100644 --- a/docs/User-Guide-Wiki.md +++ b/docs/User-Guide-Wiki.md @@ -41,10 +41,13 @@ an arbitrary release is unreliable. The escape hatches: - **Search Indexers** (the magnifier icon in the header) — free-text search across all your indexers; any result can be grabbed. -- **Manual Import** (`/import`) — point it at a folder of files you already - have and match them to books. A file with no catalogue match gets a metadata - search on its row, which creates the book (and its author, if new) and links - the file to it, so an unmatched file is no longer a dead end. +- **Import** (`/import`): **In your library** lists the books a library scan + could not match, for you to adopt in place ([Adopting files already in your + library](#adopting-files-already-in-your-library)). **From a folder** points + at a folder of files you already have and matches them to books. Either way a + file with no catalogue match gets a metadata search, which creates the book + (and its author, if new) and links the file to it, so an unmatched file is no + longer a dead end. Both still end by attaching a file to a catalogue record. @@ -67,9 +70,10 @@ Almost every "why is Bindery doing that?" question comes down to one of these. **Scan Library** (Settings → General → Library, and automatically every 6 hours) matches files already on disk to books **already in your catalogue**. -It never creates authors or books from files. If you point a fresh install at -a folder of 3,000 epubs and hit Scan, you get 3,000 "unmatched" files and an -empty library — nothing populated the catalogue first. +It never creates authors or books from files on its own. If you point a fresh +install at a folder of 3,000 epubs and hit Scan, you get a long list of +unmatched books on **Import → In your library** and an empty library, because +nothing populated the catalogue first. That list is where you can adopt them. The sequence that works is always: **populate the catalogue, then scan**. See [Bringing in an existing library](#bringing-in-an-existing-library) below. @@ -113,8 +117,9 @@ Consequences: save path Bindery hands qBittorrent. - There is no per-protocol (torrent vs usenet) folder setting because each client already owns its completed path. -- Files you acquired outside Bindery are picked up only via **Manual Import**, - **Bulk folder import**, or a **Library Scan** (rule 1 applies). +- Files you acquired outside Bindery are picked up only via **Import → From a + folder**, or a **Library Scan** and **Import → In your library** for files + already in the library folder (rule 1 applies). ### 4. Naming templates are for output, not input @@ -323,7 +328,7 @@ matches where your metadata lives, then scan: | An Audiobookshelf server | Settings → Audiobookshelf → configure + **Import** ([guide](ABS-Import-Wiki.md)) | | A Goodreads account | Settings → Import → **Goodreads CSV** (export, filter by shelf, preview, commit) | | Just a list of authors | Settings → Import → paste or upload the author list | -| Only folders of files | Use **Manual Import** (`/import`) or Settings → Import → **Bulk folder import**, which match files and create what's missing | +| Only folders of files | Scan the library, then adopt on **Import → In your library**; or use **Import → From a folder** for files outside the library | Then run **Settings → General → Library → Scan Library** to attach your files to the records. Things worth knowing before you judge the results: @@ -373,11 +378,10 @@ to the records. Things worth knowing before you judge the results: - ABS imports that "lose" titles usually didn't: ambiguous matches are parked in the **review queue** (Settings → Audiobookshelf) for you to resolve, and the import summary counts them. -- Unmatched files are listed after the scan, each with the reason it missed: - the parsed author isn't in your library (fix the file's tags or folder name), - the author matched but has no book waiting for a file (populate that author's - catalogue), no title matched, or no title could be read from the file at all - (rename it). Use **Manual Import** to resolve the rest by hand. +- Books the scan could not match wait on **Import → In your library**, one row + per book with a sentence saying why and what to do: add the author, confirm + a suggested book, or choose one. See [Adopting files already in your + library](#adopting-files-already-in-your-library). - **Fix match moves and renames the file.** When a book page shows the wrong file, the **Fix match** button reassigns it to the book you pick. That runs the full import, so the file is moved into the target book's folder and @@ -404,6 +408,85 @@ to the records. Things worth knowing before you judge the results: as evidence you own the book, and a real ebook wins over a supplement-class file when both match (#2240). +## Adopting files already in your library + +A library scan attaches every file it can match with confidence and leaves +the rest for you. Those books wait on **Import → In your library**, the page +`/import` opens on. Each row is one **book**, not one file: a 193 track +audiobook folder is one row, a folder whose audio subfolders are all discs +(`CD1`, `Disc 2`, `Disk 3`) is one row named after it, and `Dune.epub` beside +`Dune.mobi` is one row. An `Artwork` folder without audio, or a hidden or +system folder such as `@eaDir`, does not stop a disc set from grouping. +Folders named `1`, `2`, `Book 1` or `Part 1` stay separate rows; when they are +really one book, adopt each of them into it. + +**Adopting registers the files where they are.** Nothing is moved, renamed or +queued, and no indexer search starts. It is the scan's own match with you +supplying the answer, so **Undo** can take it back exactly. + +How to work through the list: + +- **Strong match** means the title is very close and the author is the same. + **Confirm** adopts it in one click. +- **Possible match** means the title is only similar, or the author differs. + Click the suggested title to check it in the editor, where it is already + selected, and adopt it from there. +- **Choose book** opens the row in place: the suggestions with their scores, + a search of your library (prefilled from the file), and a collapsed + **Search metadata**. Metadata providers are only asked when you press Search + there, so opening rows never spends provider quota. **Add and adopt** adds + the book from metadata and adopts the files in one step. +- **Books whose author is not in your library** and that share a folder are + one row: that is one decision. **Add author**, then **Scan now**, and the + scan attaches what it can by itself. Open the row to see its books, which + you can still choose or ignore one by one. Its **More** menu has **Ignore + folder** to set all of them aside. +- **More** on any row holds the rest: Choose book, Show files, Ignore. + **Ignore** hides a row that is not a book you want tracked. Later scans keep + it hidden. The **Ignored** list brings any of them back. +- **The folder list** on the left filters the table to one author folder. An + amber dot marks a folder whose author is not in your library. +- Each row says in one line what the scan found. Hovering it shows the full + explanation and the scanner's reason code, for bug reports; the editor + shows the explanation too. +- Keyboard: arrow keys move between rows, **Enter** opens one, **Esc** closes + it, **i** ignores, **u** undoes, **/** jumps to the search. + +What adopting does to your library: + +- Choosing a book **already in your library** attaches the files to it and + changes nothing else: its owner and its monitored flag stay as they were. +- A book added from metadata is added **unmonitored**, in the format you + adopted (ebook or audiobook), so Bindery never goes looking for the other + format behind your back. It and a new author are owned by the admin who + adopted them. The author's other books are not added. +- **Undo** removes exactly the file entries the adoption made, and only while + each still belongs to the book it was adopted into; a file that has since + moved to another book stays with that book. A book or author the adoption + created is removed too, unless something else now depends on it (another + file, another book by that author, excluded or not, another adopted row), + or the book has been used since: monitored, edited, linked to a series, or + searched for or downloaded. Then the book stays and Undo says so. +- If Bindery stops in the middle of an adoption, the next start reverses what + that adoption had done and the book is back in **Needs a decision**. +- Only an admin can see or act on this list, because it shows server paths. + +Things worth knowing: + +- Only regular files inside your library folders are listed. A symlink is not + adopted, including one inside the library that points elsewhere. +- A scan that finds no files at all (an unmounted volume, say) changes + nothing on this list, so your ignores and adoptions survive it. +- An adopted row stays, with Undo, for as long as its book exists. An ignored + row is forgotten 30 days after a scan last saw its files, counted only by + scans that found files in that row's library folder, or once that folder is + no longer one of your library folders at all. +- One scan lists up to 20,000 books from up to 50,000 unmatched files. A + larger library says so; adopt or ignore some and scan again. +- **From a folder** (`/import?view=folder`) is the other way in: point it at a + folder anywhere Bindery can read, such as your downloads, and it imports + what it matches into the library, moving or copying the files. + ## Metadata: where book data comes from - **OpenLibrary** is the default primary provider — it decides what an @@ -544,9 +627,10 @@ the list when you refresh the author, not on their own ([#2236](https://github.com/vavallee/bindery/issues/2236)). **Scan Library sees my files but imports nothing.** -Rule 1 — the catalogue is empty or the authors don't exist yet. Populate +Rule 1: the catalogue is empty or the authors don't exist yet. Populate first ([Bringing in an existing library](#bringing-in-an-existing-library)), -then scan. +then scan, or adopt the books from **Import → In your library** +([Adopting files](#adopting-files-already-in-your-library)). **I moved my files and a book still shows the old path.** Fixed (#2186). A book now shows whichever of its tracked files still exists, @@ -610,7 +694,9 @@ Rule 5 — separate mounts. One shared parent mount, then `auto` or `hardlink` mode. **Where do I drop files for Bindery to pick up?** -Nowhere (rule 3). Use Manual Import (`/import`) for files it didn't download. +Nowhere (rule 3). Use **Import** (`/import`) for files it didn't download: +**From a folder** for files elsewhere, **In your library** for files a library +scan found but could not match. --- diff --git a/internal/api/add_book_core.go b/internal/api/add_book_core.go new file mode 100644 index 000000000..f942da75d --- /dev/null +++ b/internal/api/add_book_core.go @@ -0,0 +1,546 @@ +package api + +import ( + "context" + "errors" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/vavallee/bindery/internal/auth" + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/indexer" + "github.com/vavallee/bindery/internal/models" +) + +// addBookParams is everything an Add Book needs, independent of HTTP. The +// AddBook handler fills it from the request body; other callers (library +// adoption, request approval) fill it themselves. Ownership is not a field: +// the core reads the acting user from ctx, exactly as the handler always has. +type addBookParams struct { + ForeignBookID string + ForeignAuthorID string + AuthorName string + SearchOnAdd bool + // MediaType optionally forces ebook, audiobook or both for the added + // book (#1397). Empty keeps the provider's media type, falling back to + // the default.media_type setting. + MediaType string + // Monitored overrides the monitored flag step 3 sets on the book. Nil + // keeps the handler's behaviour, which always marks the book monitored. + Monitored *bool + // SkipCatalogueSync suppresses the single work fallback sync that runs + // when the direct insert produced no row. With it set, the core looks + // for the row once instead of polling, since nothing it started could + // still create it. False keeps the handler's behaviour. + // + // That fallback is the only sync this core ever starts, so the skip + // removes exactly one author works call and the row it would create. + // It skips nothing else: the fallback is a single work run, which never + // refreshes the author profile (description, image), never relinks a + // Calibre author and never searches. SearchOnAdd is independent of it + // and still searches for the added book, so the two may be combined. + SkipCatalogueSync bool +} + +// addBookResult is what an Add Book produced. +type addBookResult struct { + // Book is the library row the add resolved to, after the monitored and + // media type update. + Book *models.Book + // BookCreated is true when this call inserted the row: the direct insert + // created it, or the single work fallback sync ran and the row appeared. + // False when an existing title equivalent row was adopted, or when a + // concurrent add or sync created it first. + BookCreated bool + // Author is the author the book was filed under. + Author *models.Author + // AuthorCreated is true when this call inserted the author row. + AuthorCreated bool +} + +// Refusals from addBookCore that carry no data. The AddBook handler maps +// each to the status and body it has always answered with (see +// addBookErrorResponses); other callers match them with errors.Is. +var ( + errAddBookForeignBookIDRequired = errors.New("add book: foreignBookId required") + errAddBookInvalidMediaType = errors.New("add book: invalid media type") + errAddBookAuthorMetadataUnavailable = errors.New("add book: author metadata unavailable") + errAddBookAuthorExists = errors.New("add book: author already exists") + errAddBookAuthorUnresolved = errors.New("add book: could not resolve author") + errAddBookCancelled = errors.New("add book: request cancelled") + errAddBookNotFound = errors.New("add book: book not found after author sync") + errAddBookHeldByAnotherUser = errors.New("add book: book is held by another user") +) + +// addBookErrorResponses holds the status and body for each sentinel above. +var addBookErrorResponses = []struct { + err error + status int + body string +}{ + {errAddBookForeignBookIDRequired, http.StatusBadRequest, "foreignBookId required"}, + {errAddBookInvalidMediaType, http.StatusBadRequest, "mediaType must be 'ebook', 'audiobook', or 'both'"}, + {errAddBookAuthorMetadataUnavailable, http.StatusUnprocessableEntity, "Author metadata unavailable for this result. Add the author manually first (Authors → Add Author by name), then try again."}, + {errAddBookAuthorExists, http.StatusConflict, "author already exists"}, + {errAddBookAuthorUnresolved, http.StatusInternalServerError, "could not resolve author"}, + {errAddBookCancelled, http.StatusGatewayTimeout, "request cancelled"}, + {errAddBookNotFound, http.StatusNotFound, "book not found after author sync — try again shortly"}, + {errAddBookHeldByAnotherUser, http.StatusConflict, "book is held by another user"}, +} + +// bookInLibraryError refuses to re-add a book the user can already see +// (#1227). Book is the existing row, which the handler returns in the 409. +type bookInLibraryError struct { + Book *models.Book +} + +func (e *bookInLibraryError) Error() string { return "add book: book already in library" } + +// primaryProviderUnavailableError refuses a fallback provider's record +// because the primary metadata provider did not answer (#2612). +type primaryProviderUnavailableError struct { + Primary string +} + +func (e *primaryProviderUnavailableError) Error() string { + return primaryProviderUnavailableMessage(e.Primary) +} + +// addBookLookupError wraps a failed book metadata lookup while resolving the +// author for a result that carried no author id. The handler answers 502 +// with the wrapped error's text. +type addBookLookupError struct { + Err error +} + +func (e *addBookLookupError) Error() string { return e.Err.Error() } +func (e *addBookLookupError) Unwrap() error { return e.Err } + +// writeAddBookError answers an addBookCore error with the status and body +// the AddBook handler returned before the core was split out. +func (h *AuthorHandler) writeAddBookError(w http.ResponseWriter, r *http.Request, err error) { + var inLibrary *bookInLibraryError + if errors.As(err, &inLibrary) { + h.writeExistingBookConflict(w, inLibrary.Book) + return + } + var unavailable *primaryProviderUnavailableError + if errors.As(err, &unavailable) { + writePrimaryProviderUnavailable(w, unavailable.Primary) + return + } + var lookup *addBookLookupError + if errors.As(err, &lookup) { + writeJSON(w, http.StatusBadGateway, map[string]string{"error": lookup.Err.Error()}) + return + } + for _, resp := range addBookErrorResponses { + if errors.Is(err, resp.err) { + writeJSON(w, resp.status, map[string]string{"error": resp.body}) + return + } + } + writeServerError(w, r, err) +} + +// addBookCore adds one book to the library: it resolves or creates the +// author, inserts the requested work, waits for the row, and marks it +// monitored. It is AddBook without the HTTP layer; see addBookParams for +// the inputs and the error types above for its refusals. +func (h *AuthorHandler) addBookCore(ctx context.Context, req addBookParams) (addBookResult, error) { + if req.ForeignBookID == "" { + return addBookResult{}, errAddBookForeignBookIDRequired + } + switch req.MediaType { + case "", models.MediaTypeEbook, models.MediaTypeAudiobook, models.MediaTypeBoth: + default: + return addBookResult{}, errAddBookInvalidMediaType + } + + // bookInserted and fallbackSynced record how the row came to exist, for + // addBookResult.BookCreated. + bookInserted := false + fallbackSynced := false + + // bookCreated flips true once the poll loop confirms the requested + // book is in the DB. The orphan-cleanup defer below reads it on + // AddBook return — when false (poll timeout, ctx cancel, etc.) the + // just-created author row is deleted iff it has zero books. Fixes + // issue #667 bug 3. + bookCreated := false + + // authorWasJustCreated tracks whether this request inserted the author + // row (vs. found it already present). Used by both the orphan-cleanup + // defer and the direct-insert block below — when the author was just + // created the async catalogue sync may take longer than the 15s poll + // budget for prolific authors, so we synchronously persist the requested + // book to guarantee it exists before the cleanup defer runs (#804). + authorWasJustCreated := false + + // 0. Refuse to re-add a book the user already owns (#1227). Before this + // check the handler reused the existing row and force-monitored it at + // step 3, so re-adding an owned book silently flipped it back to wanted. + // Runs before any author creation or upstream fetch so a conflict has no + // side effects. Scoped like the library list the user is looking at + // (ListScopeUserID plus NULL owners): a NULL owned row is in that list, + // and an admin under tenancy sees every row, so both are conflicts. A + // non admin's request never sees another user's copy here; the guard + // after the poll covers that case. + userID := auth.UserIDFromContext(ctx) + scopeID := auth.ListScopeUserID(ctx) + if existing, err := h.books.GetByForeignIDVisibleTo(ctx, req.ForeignBookID, scopeID); err != nil { + return addBookResult{}, err + } else if existing != nil { + return addBookResult{}, &bookInLibraryError{Book: existing} + } + + if req.ForeignAuthorID == "" { + resolved, outcome, err := h.resolveAuthorForBook(ctx, req.ForeignBookID) + if err != nil { + return addBookResult{}, &addBookLookupError{Err: err} + } + // The ISBN walk steps past a primary that timed out, so a fallback's + // record can win here only because the primary never answered. Adding + // it makes that provider the author's permanent link, and the same + // person is duplicated once the primary is back (#2117, #2612). Nothing + // has been written yet, so refuse and let a retry do the resolving. + if resolved != nil && !outcome.SafeToBind(resolved.Author.ForeignID) { + slog.Warn("AddBook: refusing to bind author to a fallback provider", + "foreignBookId", req.ForeignBookID, "primary", outcome.Primary, + "failed", outcome.FailureSummary(), "wouldHaveLinked", resolved.Author.ForeignID) + return addBookResult{}, &primaryProviderUnavailableError{Primary: outcome.Primary} + } + if resolved != nil { + // Rewrite the request so the existing fetch+poll flow targets the + // canonical provider's IDs. The user sees the canonical record (e.g. + // the OpenLibrary version) in their library; the original DNB record + // is dropped because bindery's author/book identity is single-source. + req.ForeignBookID = resolved.ForeignID + req.ForeignAuthorID = resolved.Author.ForeignID + if req.AuthorName == "" { + req.AuthorName = resolved.Author.Name + } + // The canonical id may itself already be in the library even though + // the id the client sent was not (#1227). + if existing, err := h.books.GetByForeignIDVisibleTo(ctx, req.ForeignBookID, scopeID); err != nil { + return addBookResult{}, err + } else if existing != nil { + return addBookResult{}, &bookInLibraryError{Book: existing} + } + } else if req.AuthorName != "" { + // ISBN-based resolution failed (e.g. Google Books: author name, no + // author ID, no ISBN). Resolve the author by NAME — prefer one already + // in the library so we reuse the user's existing author instead of + // duplicating it; otherwise adopt OpenLibrary's canonical record. Keep + // the chosen edition (req.ForeignBookID) — the other providers don't + // have this book. + if existing := h.findLibraryAuthorByName(ctx, req.AuthorName); existing != nil { + req.ForeignAuthorID = existing.ForeignID + } else if canonical, cErr := h.meta.ResolveCanonicalAuthor(ctx, req.AuthorName); cErr == nil && canonical != nil && outcome.SafeToBind(canonical.ForeignID) { + // The canonical lookup is OpenLibrary's, which is a fallback + // when another provider is primary, so the same guard applies. + req.ForeignAuthorID = canonical.ForeignID + } + } + if req.ForeignAuthorID == "" && outcome.PrimaryFailed { + // No provider placed the author, but the primary never answered, + // so "add the author manually" is the wrong advice: the primary + // may well know this book. + return addBookResult{}, &primaryProviderUnavailableError{Primary: outcome.Primary} + } + if req.ForeignAuthorID == "" { + return addBookResult{}, errAddBookAuthorMetadataUnavailable + } + } + + // 1. Find or create the author (unmonitored if new so we don't auto-want all books). + author, _ := h.authors.GetByForeignIDForUser(ctx, req.ForeignAuthorID, userID) + if author == nil { + author, _ = h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignAuthorID, userID) + } + if author == nil { + name := req.AuthorName + if name == "" { + name = req.ForeignAuthorID + } + fetched, err := h.meta.GetAuthor(ctx, req.ForeignAuthorID) + if err != nil || fetched == nil { + fetched = &models.Author{ + ForeignID: req.ForeignAuthorID, + Name: name, + SortName: sortName(name), + MetadataProvider: "openlibrary", + } + } + fetched.Monitored = false + def := models.DefaultMetadataProfileID + fetched.MetadataProfileID = &def + + // Dedupe path: if a canonical provider (OL / Hardcover / …) is being + // added for a SortName previously persisted as a synthetic DNB-only + // row, migrate that row in place rather than creating a duplicate. + // The synthetic row was created because the DNB record had only an + // author name (no GND link, no OL coverage). Now that a canonical + // identity exists, collapse the two onto a single primary key so + // the user keeps one author with all their books attached. + if !strings.HasPrefix(fetched.ForeignID, "dnb:") { + if existing, lookupErr := h.authors.GetByDNBSyntheticName(ctx, fetched.SortName, userID); lookupErr == nil && existing != nil { + if err := h.authors.UpgradeSyntheticDNB(ctx, existing.ForeignID, fetched); err != nil { + slog.Debug("AddBook: upgrade synthetic DNB author failed", "from", existing.ForeignID, "to", fetched.ForeignID, "error", err) + } else { + // Re-fetch the row by its new canonical ForeignID so subsequent + // steps see the upgraded record (ID preserved). + if upgraded, getErr := h.authors.GetByForeignIDForUser(ctx, fetched.ForeignID, userID); getErr == nil && upgraded != nil { + author = upgraded + } + } + } + } + + // CreateForUser may collide with a concurrent request inserting the + // same author; the UNIQUE-constraint branch below recovers by + // re-fetching the row. authorWasJustCreated stays false on the race + // path so the orphan-cleanup defer never rolls back somebody else's + // author row (issue #667). + if author == nil { + // Add-book creates the author as a side effect, so it never carries + // an explicit monitor choice — take the install-wide default (#1666). + db.ApplyAuthorMonitorDefaults(ctx, h.settings, fetched) + if err := h.authors.CreateForUser(ctx, fetched, userID); err != nil { + if !strings.Contains(err.Error(), "UNIQUE constraint failed") && !errors.Is(err, db.ErrAuthorIdentifierConflict) { + return addBookResult{}, err + } + // Race: another request created it between our check and insert. + author, _ = h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignAuthorID, userID) + if author == nil { + return addBookResult{}, errAddBookAuthorExists + } + } else { + author = fetched + authorWasJustCreated = true + // No speculative catalogue fetch here (#1816). Adding a book + // creates its author as a side effect; the user picked ONE + // title, and pulling that author's whole bibliography in behind + // it is the "my collection went from 75 books to over 500" + // report — the thing nobody expects because adding a film to + // Radarr does not import the director's filmography. + // + // Nothing downstream needs it: the direct insert below creates + // the picked book synchronously, which is what makes the poll + // succeed and what keeps the orphan-cleanup defer from + // rolling the author back. The narrow case where that insert + // cannot produce the row has its own single-work fallback, + // just past the direct-insert block. + } + } + // Defer the orphan cleanup so cancellation paths inside the poll + // loop also benefit. Runs only after a CreateForUser this request. + if authorWasJustCreated { + defer h.cleanupOrphanIfNoBooks(author, &bookCreated) + } + } + if author == nil { + return addBookResult{}, errAddBookAuthorUnresolved + } + + // 1b. Direct insert for the requested book. + // + // Originally added (#667) for DNB synthetic IDs, whose async sync returns + // zero books because DNB's public SRU has no author→works relationship. + // #804 widened this: for any author the request just created, the async + // catalogue sync can take longer than the 15 s poll budget (OpenLibrary + // took >32 s for a 175-work author in the bug report). When the poll + // times out, the orphan-cleanup defer deletes the author row — and the + // still-running goroutine then logs a FK-constraint failure for every + // book it tries to insert against the now-deleted author_id. + // + // Synchronously fetching and persisting the single requested record + // guarantees the poll succeeds on its first iteration AND that the + // cleanup defer sees a non-empty book list (so it keeps the author). + // The async sync still runs as a backfill for the rest of the catalogue; + // any UNIQUE collision against this row is silently tolerated. + // + // #1612 made the direct insert unconditional. When the author already + // EXISTED, AddBook used to skip it and rely entirely on a catalogue sync + // having created the row — but the sync can deterministically refuse a + // specific work (e.g. the work-level language sampled from the first few + // OpenLibrary editions falls outside the profile's allowed set, which is + // how heavily-translated works ended up permanently un-addable). Every + // attempt then polled 15 s for a row nothing would ever create and + // returned 404 "try again shortly" forever. An explicit add of one + // specific work is the strongest possible user signal and must not be + // vetoed by catalogue-sync heuristics; those heuristics still govern + // everything the user did NOT explicitly pick. + if existing, _ := h.books.GetByForeignID(ctx, req.ForeignBookID); existing == nil { + primary, err := h.meta.GetBook(ctx, req.ForeignBookID) + if err != nil { + slog.Warn("AddBook: direct fetch failed", + "foreignBookId", req.ForeignBookID, "error", err) + } else if primary != nil && h.directInsertTitleUsable(primary.Title, author.Name) { + primary.AuthorID = author.ID + // Tenancy (#1457): inherit the author's owner. Read off the author + // row rather than the request, because that row usually pre-exists + // this request (#1612) — the scoped lookup above is what makes it + // the correct owner either way. + primary.OwnerUserID = author.OwnerUserID + primary.Monitored = author.Monitored + if primary.Status == "" { + primary.Status = models.BookStatusWanted + } + // An explicit request choice wins over the provider's media type + // (#1397). Otherwise some providers (notably Google Books) don't + // set one; fall back to the global default so the row isn't + // created with an empty format (which would mis-route its + // indexer search). + if req.MediaType != "" { + primary.MediaType = req.MediaType + } else if primary.MediaType == "" { + primary.MediaType = h.resolveDefaultMediaType(ctx) + } + // Reuse a title-equivalent row under the same author instead of + // inserting a second one. The catalogue sync runs this same dedup + // (see the FindByAuthorAndDedupKey switch above), and skipping it + // here produced real duplicates: a Calibre-imported library holds + // the work under a `calibre:` foreign id, and OpenLibrary splits + // some works into separate ebook and audiobook Works that the sync + // merges into one media_type=both row — in both cases the + // requested foreign id has no row of its own, which is exactly the + // state that brings a user here. + match, ferr := h.books.FindByAuthorAndDedupKey(ctx, author.ID, primary.Title) + // A subtitle-collapsed dedup key (indexer.CanonicalDedupKey strips a + // ": subtitle" tail) merges every "Series: Volume" sibling onto one + // key. Adopting such a match would rebind the requested foreign id + // onto a *different* volume and — because adopt is a no-op when the + // row needs no field change — leave the poll below unable to find the + // requested id, returning 404 forever. When the requested work is a + // distinct volume of the matched row's series (same series, different + // sequence), skip the adopt and create a distinct row instead. + if ferr == nil && match != nil && !h.directInsertSeriesConflict(ctx, match.ID, primary.SeriesRefs) { + h.adoptDirectInsertMatch(ctx, match, primary, req.ForeignBookID) + } else if err := h.books.Create(ctx, primary); err != nil { + if !strings.Contains(err.Error(), "UNIQUE constraint failed") { + slog.Warn("AddBook: direct insert failed", + "foreignBookId", req.ForeignBookID, "error", err) + } + } else { + bookInserted = true + h.hydrateHardcoverEditions(ctx, primary, nil) + // Same post-create work every other creation path does + // (recommendations.go, series.go): check the library for a + // file we already have and link the book into its series. + // Without this the row is wanted-but-unchecked, so with + // searchOnAdd enabled Bindery re-downloads a book already on + // disk — the regression #940 and migration 026 exist to stop. + created := *primary + handleNewWantedBook(ctx, h.books, h.series, h.finder, created, author.Name) + } + } + } + + // 1c. Single-work fallback. The direct insert above covers the request in + // all but a couple of cases: the provider's book endpoint failed (#1612's + // OpenLibrary 502) or returned a record the title guard rejected, and the + // library has no row for the id either way. Ask the author endpoint for + // this ONE work instead — the same fetch the old speculative catalogue + // sync ran, restricted to the work the user actually picked, so the poll + // below can still succeed without the rest of the bibliography riding + // along (#1816). + if existing, _ := h.books.GetByForeignID(ctx, req.ForeignBookID); existing == nil && !req.SkipCatalogueSync { + // mediaType only fills a format the provider left blank, and step 3 + // below applies the request's explicit choice to whatever row the poll + // finds — so the default is the right value to pass here. It no longer + // decides whether the work is created at all: a single-work run is + // exempt from the strict media-type clamp (#1612). + fallbackSynced = true + h.fetchAuthorBooksAsync(author, catalogueSyncOptions{ + mediaType: h.resolveDefaultMediaType(ctx), + onlyForeignID: req.ForeignBookID, + }) + } + + // 2. Poll until the book appears (the single-work fallback, if it ran, + // creates it asynchronously). + deadline := time.Now().Add(15 * time.Second) + if req.SkipCatalogueSync { + deadline = time.Now() + } + var book *models.Book + for { + b, _ := h.books.GetByForeignID(ctx, req.ForeignBookID) + if b != nil { + book = b + break + } + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return addBookResult{}, errAddBookCancelled + case <-time.After(500 * time.Millisecond): + } + } + + if book == nil { + return addBookResult{}, errAddBookNotFound + } + // The poll looks the row up globally because books.foreign_id is UNIQUE + // across users, so under tenancy it can return another user's copy: the + // conflict gate above never saw it (scoped to this user), the direct + // insert skipped it, and until #1227 the update below then flipped that + // user's book to monitored. Refuse instead, with no row in the body since + // the caller is not allowed to see it. bookCreated stays false so the + // orphan cleanup defer removes any author row this request created. + if book.OwnerUserID != 0 && userID != 0 && book.OwnerUserID != userID { + return addBookResult{}, errAddBookHeldByAnotherUser + } + bookCreated = true + + // 3. Mark the book monitored (wanted). An explicit media-type choice is + // applied here too — the poll may have found a row created by the async + // catalogue sync (or one already in the library) carrying the default. + // Re-evaluate status on a change so e.g. adding an already-imported ebook + // as 'both' flips it back to wanted for the missing format (#1148). + book.Monitored = true + if req.Monitored != nil { + book.Monitored = *req.Monitored + } + if req.MediaType != "" && book.MediaType != req.MediaType { + book.MediaType = req.MediaType + reevaluateBookStatus(book) + } + if err := h.books.Update(ctx, book); err != nil { + return addBookResult{}, err + } + + // 3b. Say so when this add went past the strict media-type policy (#1759). + // + // The policy is a catalogue-population rule, not a veto on what the user + // may own: the direct insert above never consults it, and the single-work + // fallback is exempt by #1612's rule that "an explicit add of one specific + // work must not be vetoed by catalogue-sync heuristics". Both of those are + // deliberate, because silently refusing an explicit user action is the + // worse of the two failures. + // + // What was missing is that it happened invisibly, so a user who turned the + // setting on to stop un-grabbable rows appearing had no way to learn that + // their own add was the exception. The setting's help text now says the + // same thing, which is the half most people will actually see. + h.logStrictMediaTypeBypass(ctx, book) + + // 4. Optionally trigger an indexer search. Use the process-lifecycle + // context so the search goroutine is cancelled on shutdown rather than + // running against context.Background(). See #846. + if req.SearchOnAdd && h.searcher != nil { + go h.searcher.SearchAndGrabBook(indexer.WithSearchOrigin(h.bgCtx(), indexer.OriginAuthor), *book) // #nosec G118 -- intentional: search must outlive the request + } + + return addBookResult{ + Book: book, + BookCreated: bookInserted || fallbackSynced, + Author: author, + AuthorCreated: authorWasJustCreated, + }, nil +} diff --git a/internal/api/add_book_core_test.go b/internal/api/add_book_core_test.go new file mode 100644 index 000000000..1f44fb425 --- /dev/null +++ b/internal/api/add_book_core_test.go @@ -0,0 +1,467 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/jobs" + "github.com/vavallee/bindery/internal/metadata" + "github.com/vavallee/bindery/internal/models" +) + +// worksCountingProvider counts the author works calls a catalogue sync makes, +// so a test can tell whether addBookCore or createAuthorCore started one. +type worksCountingProvider struct { + *stubMetaProvider + worksCalls atomic.Int32 +} + +func (p *worksCountingProvider) GetAuthorWorks(ctx context.Context, id string) ([]models.Book, error) { + p.worksCalls.Add(1) + return p.stubMetaProvider.GetAuthorWorks(ctx, id) +} + +func (p *worksCountingProvider) GetAuthorWorksByName(ctx context.Context, name string) ([]models.Book, error) { + p.worksCalls.Add(1) + return p.stubMetaProvider.GetAuthorWorksByName(ctx, name) +} + +type coreFixture struct { + h *AuthorHandler + authors *db.AuthorRepo + books *db.BookRepo + jobs *jobs.Group +} + +// newCoreFixture wires an AuthorHandler over a fresh in memory database with +// a jobs group, so a test can drain every catalogue sync the call started +// before it counts provider calls. +func newCoreFixture(t *testing.T, provider metadata.Provider) coreFixture { + t.Helper() + database, err := db.OpenMemory() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + database.SetMaxOpenConns(1) + + authorRepo := db.NewAuthorRepo(database) + bookRepo := db.NewBookRepo(database) + profileRepo := db.NewMetadataProfileRepo(database) + group := jobs.NewGroup(context.Background()) + t.Cleanup(func() { group.Shutdown(5 * time.Second) }) + h := NewAuthorHandler(authorRepo, nil, bookRepo, nil, + metadata.NewAggregator(provider), nil, profileRepo, nil).WithJobs(group) + return coreFixture{h: h, authors: authorRepo, books: bookRepo, jobs: group} +} + +// drain waits for every job the handler started and fails on a straggler. +func (f coreFixture) drain(t *testing.T) { + t.Helper() + if left := f.jobs.Shutdown(5 * time.Second); len(left) != 0 { + t.Fatalf("jobs still running after drain: %v", left) + } +} + +func seedWells(t *testing.T, authors *db.AuthorRepo) *models.Author { + t.Helper() + author := &models.Author{ + ForeignID: "OL39307A", Name: "H. G. Wells", SortName: "Wells, H. G.", + MetadataProvider: "openlibrary", Monitored: true, + MonitorMode: models.AuthorMonitorModeAll, + } + if err := authors.Create(context.Background(), author); err != nil { + t.Fatal(err) + } + return author +} + +var wellsAddBookParams = addBookParams{ + ForeignBookID: "OL27482W", + ForeignAuthorID: "OL39307A", + AuthorName: "H. G. Wells", +} + +// Default params must give what the handler gives for the same request: the +// same row, monitored, filed under a newly created author. +func TestAddBookCore_DefaultParamsMatchHandler(t *testing.T) { + viaHandler := newCoreFixture(t, addBookBackCatalogueStub(false)) + body, _ := json.Marshal(map[string]any{ + "foreignBookId": wellsAddBookParams.ForeignBookID, + "foreignAuthorId": wellsAddBookParams.ForeignAuthorID, + "authorName": wellsAddBookParams.AuthorName, + }) + rec := httptest.NewRecorder() + viaHandler.h.AddBook(rec, httptest.NewRequest(http.MethodPost, "/api/v1/author/book", bytes.NewReader(body))) + if rec.Code != http.StatusCreated { + t.Fatalf("handler: expected 201, got %d: %s", rec.Code, rec.Body.String()) + } + var want models.Book + if err := json.Unmarshal(rec.Body.Bytes(), &want); err != nil { + t.Fatal(err) + } + + viaCore := newCoreFixture(t, addBookBackCatalogueStub(false)) + res, err := viaCore.h.addBookCore(context.Background(), wellsAddBookParams) + if err != nil { + t.Fatalf("core: %v", err) + } + got := res.Book + if got == nil { + t.Fatal("core returned no book") + } + if got.ForeignID != want.ForeignID || got.Title != want.Title || got.Monitored != want.Monitored || + got.MediaType != want.MediaType || got.Status != want.Status || got.OwnerUserID != want.OwnerUserID { + t.Errorf("core book = %+v\nhandler book = %+v", *got, want) + } + if !got.Monitored { + t.Error("default params must leave the book monitored") + } + if !res.BookCreated || !res.AuthorCreated { + t.Errorf("BookCreated=%v AuthorCreated=%v, want both true for a new author and book", res.BookCreated, res.AuthorCreated) + } + if res.Author == nil || res.Author.ForeignID != "OL39307A" || got.AuthorID != res.Author.ID { + t.Errorf("result author = %+v, book author id %d", res.Author, got.AuthorID) + } + stored, err := viaCore.books.GetByForeignID(context.Background(), "OL27482W") + if err != nil || stored == nil || stored.ID != got.ID || !stored.Monitored { + t.Errorf("stored row = %+v err=%v, want the returned monitored row", stored, err) + } + + // A second add of the same book is refused with the existing row, which + // the handler turns into its 409. + _, err = viaCore.h.addBookCore(context.Background(), wellsAddBookParams) + var inLibrary *bookInLibraryError + if !errors.As(err, &inLibrary) || inLibrary.Book == nil || inLibrary.Book.ID != got.ID { + t.Fatalf("re-add error = %v, want bookInLibraryError for row %d", err, got.ID) + } + rec = httptest.NewRecorder() + viaCore.h.writeAddBookError(rec, httptest.NewRequest(http.MethodPost, "/api/v1/author/book", nil), err) + if rec.Code != http.StatusConflict { + t.Errorf("mapped re-add status = %d, want 409", rec.Code) + } +} + +func TestAddBookCore_ExistingAuthorIsNotReportedCreated(t *testing.T) { + f := newCoreFixture(t, addBookBackCatalogueStub(false)) + author := seedWells(t, f.authors) + + res, err := f.h.addBookCore(context.Background(), wellsAddBookParams) + if err != nil { + t.Fatal(err) + } + if res.AuthorCreated { + t.Error("AuthorCreated = true for an author already in the library") + } + if !res.BookCreated { + t.Error("BookCreated = false for a book the direct insert created") + } + if res.Author == nil || res.Author.ID != author.ID { + t.Errorf("result author = %+v, want id %d", res.Author, author.ID) + } +} + +func TestAddBookCore_MonitoredFalseCreatesUnmonitoredBook(t *testing.T) { + f := newCoreFixture(t, addBookBackCatalogueStub(false)) + p := wellsAddBookParams + unmonitored := false + p.Monitored = &unmonitored + + res, err := f.h.addBookCore(context.Background(), p) + if err != nil { + t.Fatal(err) + } + if res.Book.Monitored { + t.Error("returned book is monitored, want unmonitored") + } + stored, err := f.books.GetByForeignID(context.Background(), "OL27482W") + if err != nil || stored == nil { + t.Fatalf("book not stored: err=%v", err) + } + if stored.Monitored { + t.Error("stored book is monitored, want unmonitored") + } + if !res.BookCreated { + t.Error("BookCreated = false for a newly inserted book") + } +} + +// When the direct insert fails, the default path falls back to a single work +// catalogue sync. SkipCatalogueSync must start no sync and answer at once +// instead of polling the full 15 seconds. +func TestAddBookCore_SkipCatalogueSyncFiresNoFetch(t *testing.T) { + t.Run("default runs the fallback sync", func(t *testing.T) { + provider := &worksCountingProvider{stubMetaProvider: addBookBackCatalogueStub(true)} + f := newCoreFixture(t, provider) + seedWells(t, f.authors) + + res, err := f.h.addBookCore(context.Background(), wellsAddBookParams) + if err != nil { + t.Fatalf("expected the fallback to create the book, got %v", err) + } + f.drain(t) + if provider.worksCalls.Load() == 0 { + t.Error("fallback made no author works call; the control case proves nothing") + } + if !res.BookCreated { + t.Error("BookCreated = false for a book the fallback sync created") + } + }) + + t.Run("skip starts no sync", func(t *testing.T) { + provider := &worksCountingProvider{stubMetaProvider: addBookBackCatalogueStub(true)} + f := newCoreFixture(t, provider) + seedWells(t, f.authors) + p := wellsAddBookParams + p.SkipCatalogueSync = true + + start := time.Now() + _, err := f.h.addBookCore(context.Background(), p) + elapsed := time.Since(start) + if !errors.Is(err, errAddBookNotFound) { + t.Fatalf("err = %v, want errAddBookNotFound", err) + } + f.drain(t) + if n := provider.worksCalls.Load(); n != 0 { + t.Errorf("author works calls = %d, want 0", n) + } + if got, _ := f.books.GetByForeignID(context.Background(), "OL27482W"); got != nil { + t.Errorf("book created without a sync: %+v", got) + } + if elapsed > 5*time.Second { + t.Errorf("skip path took %v; it should not poll", elapsed) + } + }) +} + +func wellsCreateProvider() *worksCountingProvider { + stub := addBookBackCatalogueStub(false) + stub.author = &models.Author{ + ForeignID: "OL39307A", Name: "H. G. Wells", SortName: "Wells, H. G.", + MetadataProvider: "openlibrary", + } + return &worksCountingProvider{stubMetaProvider: stub} +} + +var wellsCreateParams = createAuthorParams{ + ForeignID: "OL39307A", + Name: "H. G. Wells", + Monitored: true, +} + +func TestCreateAuthorCore_DefaultParamsMatchHandler(t *testing.T) { + viaHandler := newCoreFixture(t, wellsCreateProvider()) + body, _ := json.Marshal(map[string]any{ + "foreignAuthorId": wellsCreateParams.ForeignID, + "authorName": wellsCreateParams.Name, + "monitored": wellsCreateParams.Monitored, + }) + rec := httptest.NewRecorder() + viaHandler.h.Create(rec, httptest.NewRequest(http.MethodPost, "/api/v1/author", bytes.NewReader(body))) + if rec.Code != http.StatusCreated { + t.Fatalf("handler: expected 201, got %d: %s", rec.Code, rec.Body.String()) + } + var want models.Author + if err := json.Unmarshal(rec.Body.Bytes(), &want); err != nil { + t.Fatal(err) + } + viaHandler.drain(t) + + provider := wellsCreateProvider() + viaCore := newCoreFixture(t, provider) + res, err := viaCore.h.createAuthorCore(context.Background(), wellsCreateParams) + if err != nil { + t.Fatalf("core: %v", err) + } + viaCore.drain(t) + got := res.Author + if !res.Created || got == nil { + t.Fatalf("result = %+v, want a created author", res) + } + if got.ForeignID != want.ForeignID || got.Name != want.Name || got.Monitored != want.Monitored || + got.MonitorMode != want.MonitorMode || got.MonitorNewItems != want.MonitorNewItems { + t.Errorf("core author = %+v\nhandler author = %+v", *got, want) + } + if provider.worksCalls.Load() == 0 { + t.Error("default params started no catalogue sync") + } + + // Adding the same author again is the handler's 409 with the canonical row. + _, err = viaCore.h.createAuthorCore(context.Background(), wellsCreateParams) + var conflict *authorConflictError + if !errors.As(err, &conflict) || conflict.Canonical == nil || conflict.Canonical.ID != got.ID { + t.Fatalf("re-create error = %v, want authorConflictError for author %d", err, got.ID) + } +} + +func TestCreateAuthorCore_SkipCatalogueSyncFiresNoFetch(t *testing.T) { + provider := wellsCreateProvider() + f := newCoreFixture(t, provider) + p := wellsCreateParams + p.SkipCatalogueSync = true + + res, err := f.h.createAuthorCore(context.Background(), p) + if err != nil { + t.Fatal(err) + } + f.drain(t) + if !res.Created { + t.Error("Created = false, want a new author") + } + if n := provider.worksCalls.Load(); n != 0 { + t.Errorf("author works calls = %d, want 0", n) + } + books, err := f.books.ListByAuthor(context.Background(), res.Author.ID) + if err != nil { + t.Fatal(err) + } + if len(books) != 0 { + t.Errorf("skip created %d books, want 0", len(books)) + } +} + +// errorResponseCase is one row of the error mapping tables below. body holds +// the exact JSON object the handler must write, keys and values both. +type errorResponseCase struct { + name string + err error + status int + body map[string]any +} + +func assertErrorResponses(t *testing.T, write func(http.ResponseWriter, *http.Request, error), cases []errorResponseCase) { + t.Helper() + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := httptest.NewRecorder() + write(rec, httptest.NewRequest(http.MethodPost, "/api/v1/test", nil), tc.err) + if rec.Code != tc.status { + t.Errorf("status = %d, want %d", rec.Code, tc.status) + } + var got any + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("body %q: %v", rec.Body.String(), err) + } + // Round trip the expectation through JSON so numbers and nested + // structs compare in the same shape the handler's body decodes to. + wantRaw, err := json.Marshal(tc.body) + if err != nil { + t.Fatal(err) + } + var want any + if err := json.Unmarshal(wantRaw, &want); err != nil { + t.Fatal(err) + } + gotJSON, _ := json.Marshal(got) + wantJSON, _ := json.Marshal(want) + if string(gotJSON) != string(wantJSON) { + t.Errorf("body = %s\nwant %s", gotJSON, wantJSON) + } + }) + } +} + +func errorBody(msg string) map[string]any { return map[string]any{"error": msg} } + +// Every error addBookCore can return, with the literal status and body the +// AddBook handler answered with before the split. The bodies are spelled out +// here rather than read from addBookErrorResponses, so a change to that table +// fails this test. +func TestWriteAddBookError_Statuses(t *testing.T) { + h := &AuthorHandler{} + existing := &models.Book{ID: 42, ForeignID: "OL1W", Title: "Kept"} + assertErrorResponses(t, h.writeAddBookError, []errorResponseCase{ + {"foreign book id required", errAddBookForeignBookIDRequired, http.StatusBadRequest, + errorBody("foreignBookId required")}, + {"invalid media type", errAddBookInvalidMediaType, http.StatusBadRequest, + errorBody("mediaType must be 'ebook', 'audiobook', or 'both'")}, + {"author metadata unavailable", errAddBookAuthorMetadataUnavailable, http.StatusUnprocessableEntity, + errorBody("Author metadata unavailable for this result. Add the author manually first (Authors → Add Author by name), then try again.")}, + {"author exists", errAddBookAuthorExists, http.StatusConflict, + errorBody("author already exists")}, + {"author unresolved", errAddBookAuthorUnresolved, http.StatusInternalServerError, + errorBody("could not resolve author")}, + {"cancelled", errAddBookCancelled, http.StatusGatewayTimeout, + errorBody("request cancelled")}, + {"not found after sync", errAddBookNotFound, http.StatusNotFound, + errorBody("book not found after author sync — try again shortly")}, + {"held by another user", errAddBookHeldByAnotherUser, http.StatusConflict, + errorBody("book is held by another user")}, + {"book in library", &bookInLibraryError{Book: existing}, http.StatusConflict, map[string]any{ + "error": "book already in your library; change its format or monitoring from the book page", + "existingBookId": existing.ID, + "existingBook": existing, + }}, + {"primary provider unavailable", &primaryProviderUnavailableError{Primary: "hardcover"}, http.StatusServiceUnavailable, + errorBody("The primary metadata provider (hardcover) did not answer, so no record from another provider was used. Please try again once it responds.")}, + {"book lookup failed", &addBookLookupError{Err: errors.New("look up book metadata: boom")}, http.StatusBadGateway, + errorBody("look up book metadata: boom")}, + {"unknown error", errors.New("disk on fire"), http.StatusInternalServerError, + errorBody("internal server error")}, + }) + // A sentinel added to the table without a row above would go unchecked. + if len(addBookErrorResponses) != 8 { + t.Errorf("addBookErrorResponses has %d rows, this test covers 8; add a case for the new one", len(addBookErrorResponses)) + } +} + +// Every error createAuthorCore can return, with the literal status and body +// the Create handler answered with before the split. +func TestWriteCreateAuthorError_Statuses(t *testing.T) { + h := &AuthorHandler{} + canonical := &models.Author{ID: 7, ForeignID: "OL7A", Name: "Canon"} + assertErrorResponses(t, h.writeCreateAuthorError, []errorResponseCase{ + {"fields required", errCreateAuthorFieldsRequired, http.StatusBadRequest, + errorBody("foreignAuthorId and authorName required")}, + {"invalid monitor new items", errCreateAuthorInvalidMonitorNewItems, http.StatusBadRequest, + errorBody("invalid monitorNewItems")}, + {"search needs sync", errCreateAuthorSearchNeedsSync, http.StatusBadRequest, + errorBody("searchOnAdd requires the catalogue sync")}, + {"invalid monitor option", &createAuthorOptionError{Err: errors.New("monitorMode must be one of: all, future, latest, none")}, http.StatusBadRequest, + errorBody("monitorMode must be one of: all, future, latest, none")}, + {"upstream lookup failed", &createAuthorLookupError{Err: errors.New("openlibrary: upstream 502")}, http.StatusBadGateway, + errorBody("openlibrary: upstream 502")}, + {"conflict with canonical author", &authorConflictError{Canonical: canonical, Message: "author already exists"}, http.StatusConflict, map[string]any{ + "error": "author already exists", + "canonicalAuthorId": canonical.ID, + "canonicalAuthor": canonical, + }}, + {"conflict without canonical author", &authorConflictError{Message: "author name resolves ambiguously — merge manually"}, http.StatusConflict, + errorBody("author name resolves ambiguously — merge manually")}, + {"unknown error", errors.New("disk on fire"), http.StatusInternalServerError, + errorBody("internal server error")}, + }) + if len(createAuthorErrorResponses) != 3 { + t.Errorf("createAuthorErrorResponses has %d rows, this test covers 3; add a case for the new one", len(createAuthorErrorResponses)) + } +} + +// SearchOnAdd only ever runs inside the catalogue sync, so asking for it with +// the sync skipped is refused before anything is written. +func TestCreateAuthorCore_SearchOnAddWithSkipIsRefused(t *testing.T) { + provider := wellsCreateProvider() + f := newCoreFixture(t, provider) + p := wellsCreateParams + p.SkipCatalogueSync = true + p.SearchOnAdd = true + + _, err := f.h.createAuthorCore(context.Background(), p) + if !errors.Is(err, errCreateAuthorSearchNeedsSync) { + t.Fatalf("err = %v, want errCreateAuthorSearchNeedsSync", err) + } + f.drain(t) + if got, _ := f.authors.GetByForeignID(context.Background(), "OL39307A"); got != nil { + t.Errorf("refused call still created author %+v", got) + } + if n := provider.worksCalls.Load(); n != 0 { + t.Errorf("author works calls = %d, want 0", n) + } +} diff --git a/internal/api/adoption.go b/internal/api/adoption.go new file mode 100644 index 000000000..bd850c714 --- /dev/null +++ b/internal/api/adoption.go @@ -0,0 +1,476 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/importer" +) + +// Library adoption (/library/unmatched): the books a library scan could not +// match, and the decisions an admin makes about them. Every route is admin +// only (registerAdoptionRoutes). No request carries a path: rows are addressed +// by id and every path is resolved on the server (S9, S10). + +// maxAdoptionMembersShown bounds the file names a row carries for display. +const maxAdoptionMembersShown = 20 + +// adoptionScanner is the part of the scanner the adoption list reads. +type adoptionScanner interface { + ScanRunning() bool +} + +// adoptionBookAdder is the add book core (see add_book_core.go). +type adoptionBookAdder interface { + addBookCore(ctx context.Context, req addBookParams) (addBookResult, error) +} + +// adoptionUnitStore is the unmatched unit repository surface the handler uses +// (db.UnmatchedUnitRepo). An interface so a test can count the queries a page +// costs. +type adoptionUnitStore interface { + List(ctx context.Context, q db.UnmatchedListQuery) ([]db.UnmatchedUnit, int, error) + Facets(ctx context.Context, q db.UnmatchedListQuery) (db.UnmatchedFacets, error) + Summary(ctx context.Context) (db.UnmatchedSummary, error) + Get(ctx context.Context, id int64) (*db.UnmatchedUnit, error) + BookRefs(ctx context.Context, ids []int64) (map[int64]db.UnmatchedBookRef, error) + ClaimState(ctx context.Context, id int64, from, to string) (bool, error) + Claim(ctx context.Context, id int64, from, to string) (string, error) + ReleaseClaim(ctx context.Context, id int64, from, to, token string) (bool, error) + RecordAdoptionProgress(ctx context.Context, id int64, token string, rec db.AdoptionRecord) (bool, error) + CompleteAdoption(ctx context.Context, id int64, token string, rec db.AdoptionRecord) (bool, error) + CompleteUndo(ctx context.Context, id int64, token string) (bool, error) + ResetToPending(ctx context.Context, id int64, from, token string) (bool, error) + StaleClaims(ctx context.Context, before time.Time) ([]db.UnmatchedUnit, error) + BookFingerprint(ctx context.Context, bookID int64) (string, error) + IgnorePending(ctx context.Context, ids []int64, authorFolder string) (int64, error) + BookReferencedElsewhere(ctx context.Context, bookID, exceptID int64) (bool, error) + AuthorReferencedElsewhere(ctx context.Context, authorID, exceptID int64) (bool, error) +} + +// AdoptionHandler serves the adoption view on the Import page. +type AdoptionHandler struct { + units adoptionUnitStore + books *db.BookRepo + authors *db.AuthorRepo + adder adoptionBookAdder + roots *LibraryRoots + scanner adoptionScanner + settings *db.SettingsRepo + // registerFile records one file against a book and reports whether this + // call inserted it. A seam so tests can fail registration part way. + registerFile func(ctx context.Context, bookID int64, format, path string) (bool, error) + // untrackFile removes one registered row while it still belongs to bookID. + // A seam so tests can fail a reversal part way. + untrackFile func(ctx context.Context, path string, bookID int64) (bool, error) +} + +// NewAdoptionHandler wires the adoption routes. roots must be the library +// roots (not the download directories): adoption only ever registers files +// that already live in the library. +func NewAdoptionHandler(units *db.UnmatchedUnitRepo, books *db.BookRepo, authors *db.AuthorRepo, + authorHandler *AuthorHandler, roots *LibraryRoots, scanner *importer.Scanner, settings *db.SettingsRepo) *AdoptionHandler { + h := &AdoptionHandler{units: units, books: books, authors: authors, roots: roots, settings: settings} + if authorHandler != nil { + h.adder = authorHandler + } + if scanner != nil { + h.scanner = scanner + } + h.registerFile = books.AddBookFileIfMissing + h.untrackFile = books.UntrackFilePathForBook + return h +} + +// adoptionCandidate is one suggestion with its book. +type adoptionCandidate struct { + Book db.UnmatchedBookRef `json:"book"` + Score float64 `json:"score"` +} + +// adoptionItem is one row as the web sees it. +type adoptionItem struct { + ID int64 `json:"id"` + Kind string `json:"kind"` + Format string `json:"format"` + FileCount int `json:"fileCount"` + SizeBytes int64 `json:"sizeBytes"` + RelPath string `json:"relPath"` + RootPath string `json:"rootPath"` + AuthorFolder string `json:"authorFolder"` + ParsedTitle string `json:"parsedTitle"` + ParsedAuthor string `json:"parsedAuthor"` + Reason string `json:"reason"` + Candidates []adoptionCandidate `json:"candidates"` + TopScore float64 `json:"topScore"` + State string `json:"state"` + Book *db.UnmatchedBookRef `json:"book,omitempty"` + // BookCreated and AuthorCreated say whether the adoption added them, which + // is also what Undo would remove. + BookCreated bool `json:"bookCreated"` + AuthorCreated bool `json:"authorCreated"` + Members []string `json:"members"` + FirstSeenAt time.Time `json:"firstSeenAt"` + ResolvedAt *time.Time `json:"resolvedAt,omitempty"` + // Message explains an outcome that is not the obvious one, such as Undo + // keeping a book that has been used since it was added. + Message string `json:"message,omitempty"` +} + +// adoptionScanStatus is what the page says about the last and current scan. +type adoptionScanStatus struct { + Ran bool `json:"ran"` + RanAt string `json:"ranAt,omitempty"` + Running bool `json:"running"` + FilesFound int `json:"filesFound"` + Truncated bool `json:"truncated"` + Error string `json:"error,omitempty"` + NoFilesFound bool `json:"noFilesFound"` +} + +type adoptionListResponse struct { + Items []adoptionItem `json:"items"` + Total int `json:"total"` + Facets *db.UnmatchedFacets `json:"facets,omitempty"` + Summary db.UnmatchedSummary `json:"summary"` + Scan adoptionScanStatus `json:"scan"` +} + +type adoptionSummaryResponse struct { + db.UnmatchedSummary + Scan adoptionScanStatus `json:"scan"` +} + +// List handles GET /library/unmatched. Facets are computed only when asked +// for (facets=1), which the web does when the state or search changes, not +// on every page turn (P4). Candidates and adopted books for the whole page +// are hydrated in one query. +func (h *AdoptionHandler) List(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + // Release rows a request died holding, reversing what it did, so they do + // not sit invisible. Cheap: an indexed state lookup that is usually empty. + if _, err := h.RecoverStaleClaims(context.WithoutCancel(ctx), db.UnmatchedClaimTimeout); err != nil { + slog.Warn("adoption: stale claim recovery failed", "error", err) + } + qv := r.URL.Query() + limit, _ := strconv.Atoi(qv.Get("limit")) + offset, _ := strconv.Atoi(qv.Get("offset")) + q := db.UnmatchedListQuery{ + State: qv.Get("state"), + Reason: qv.Get("reason"), + AuthorFolder: qv.Get("authorFolder"), + Format: qv.Get("format"), + Search: qv.Get("search"), + Sort: qv.Get("sort"), + Dir: qv.Get("dir"), + Limit: limit, + Offset: offset, + } + units, total, err := h.units.List(ctx, q) + if err != nil { + writeServerError(w, r, err) + return + } + items, err := h.hydrate(ctx, units) + if err != nil { + writeServerError(w, r, err) + return + } + resp := adoptionListResponse{Items: items, Total: total, Scan: h.scanStatus(ctx)} + if qv.Get("facets") == "1" { + f, err := h.units.Facets(ctx, q) + if err != nil { + writeServerError(w, r, err) + return + } + resp.Facets = &f + } + if resp.Summary, err = h.units.Summary(ctx); err != nil { + writeServerError(w, r, err) + return + } + writeJSON(w, http.StatusOK, resp) +} + +// Summary handles GET /library/unmatched/summary: the counts behind the nav +// badge and the summary strip, and the scan status the page polls while a +// scan runs (P5). +func (h *AdoptionHandler) Summary(w http.ResponseWriter, r *http.Request) { + sum, err := h.units.Summary(r.Context()) + if err != nil { + writeServerError(w, r, err) + return + } + writeJSON(w, http.StatusOK, adoptionSummaryResponse{UnmatchedSummary: sum, Scan: h.scanStatus(r.Context())}) +} + +// hydrate turns rows into items, loading every book they point at at once. +func (h *AdoptionHandler) hydrate(ctx context.Context, units []db.UnmatchedUnit) ([]adoptionItem, error) { + seen := make(map[int64]bool) + var ids []int64 + want := func(id int64) { + if id > 0 && !seen[id] { + seen[id] = true + ids = append(ids, id) + } + } + for i := range units { + want(units[i].BookID) + for _, c := range units[i].Candidates { + want(c.BookID) + } + } + refs, err := h.units.BookRefs(ctx, ids) + if err != nil { + return nil, err + } + items := make([]adoptionItem, len(units)) + for i := range units { + items[i] = toAdoptionItem(&units[i], refs) + } + return items, nil +} + +func toAdoptionItem(u *db.UnmatchedUnit, refs map[int64]db.UnmatchedBookRef) adoptionItem { + it := adoptionItem{ + ID: u.ID, Kind: u.UnitKind, Format: u.Format, FileCount: u.FileCount, SizeBytes: u.SizeBytes, + RelPath: u.RelPath, RootPath: u.RootPath, AuthorFolder: u.AuthorFolder, + ParsedTitle: u.ParsedTitle, ParsedAuthor: u.ParsedAuthor, Reason: u.Reason, + Candidates: []adoptionCandidate{}, TopScore: u.TopScore, State: u.State, + BookCreated: u.CreatedBookID > 0, AuthorCreated: u.CreatedAuthorID > 0, + Members: memberNames(u), FirstSeenAt: u.FirstSeenAt, ResolvedAt: u.ResolvedAt, + } + for _, c := range u.Candidates { + // A suggested book deleted since the scan simply drops out. + if ref, ok := refs[c.BookID]; ok { + it.Candidates = append(it.Candidates, adoptionCandidate{Book: ref, Score: c.Score}) + } + } + if ref, ok := refs[u.BookID]; ok && u.BookID > 0 { + it.Book = &ref + } + return it +} + +// memberNames lists up to maxAdoptionMembersShown member files, relative to +// the unit folder, or by base name for a file unit. +func memberNames(u *db.UnmatchedUnit) []string { + n := min(len(u.MemberPaths), maxAdoptionMembersShown) + out := make([]string, 0, n) + for _, p := range u.MemberPaths[:n] { + name := filepath.Base(p) + if u.UnitKind == db.UnmatchedKindFolder { + if rel, err := filepath.Rel(u.UnitPath, p); err == nil { + name = filepath.ToSlash(rel) + } + } + out = append(out, name) + } + return out +} + +// scanStatus reads the last scan result blob and the live running flag. +func (h *AdoptionHandler) scanStatus(ctx context.Context) adoptionScanStatus { + st := adoptionScanStatus{} + if h.scanner != nil { + st.Running = h.scanner.ScanRunning() + } + if h.settings == nil { + return st + } + setting, err := h.settings.Get(ctx, SettingLibraryLastScan) + if err != nil || setting == nil || setting.Value == "" { + return st + } + var blob struct { + RanAt string `json:"ran_at"` + FilesFound int `json:"files_found"` + UnitsTruncated bool `json:"units_truncated"` + ScanError string `json:"scan_error"` + NoFilesFound bool `json:"no_files_found"` + } + if json.Unmarshal([]byte(setting.Value), &blob) != nil { + return st + } + st.Ran = true + st.RanAt, st.FilesFound, st.Truncated = blob.RanAt, blob.FilesFound, blob.UnitsTruncated + st.Error, st.NoFilesFound = blob.ScanError, blob.NoFilesFound + return st +} + +// unitIDParam reads {id}, answering 400 itself when it is not a number. +func unitIDParam(w http.ResponseWriter, r *http.Request) (int64, bool) { + id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil || id <= 0 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid id"}) + return 0, false + } + return id, true +} + +// writeUnit answers with one row, freshly read and hydrated. +func (h *AdoptionHandler) writeUnit(w http.ResponseWriter, r *http.Request, id int64) { + h.writeUnitWithMessage(w, r, id, "") +} + +func (h *AdoptionHandler) writeUnitWithMessage(w http.ResponseWriter, r *http.Request, id int64, message string) { + u, err := h.units.Get(r.Context(), id) + if err != nil { + writeServerError(w, r, err) + return + } + if u == nil { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "unmatched book not found"}) + return + } + items, err := h.hydrate(r.Context(), []db.UnmatchedUnit{*u}) + if err != nil { + writeServerError(w, r, err) + return + } + items[0].Message = message + writeJSON(w, http.StatusOK, items[0]) +} + +// stateConflict answers a compare and swap that lost, naming the state the +// row is actually in so the web can show something truthful. +func (h *AdoptionHandler) stateConflict(w http.ResponseWriter, r *http.Request, id int64) { + u, err := h.units.Get(r.Context(), id) + if err != nil { + writeServerError(w, r, err) + return + } + if u == nil { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "unmatched book not found"}) + return + } + msg := map[string]string{ + db.UnmatchedStatePending: "This book is still waiting for a decision.", + db.UnmatchedStateAdopting: "Someone is adopting this book right now.", + db.UnmatchedStateAdopted: "This book was already adopted.", + db.UnmatchedStateUndoing: "Someone is undoing this adoption right now.", + db.UnmatchedStateIgnored: "This book is ignored.", + }[u.State] + writeJSON(w, http.StatusConflict, map[string]string{"error": msg, "state": u.State}) +} + +// Ignore handles POST /library/unmatched/{id}/ignore. +func (h *AdoptionHandler) Ignore(w http.ResponseWriter, r *http.Request) { + h.moveState(w, r, db.UnmatchedStatePending, db.UnmatchedStateIgnored) +} + +// Unignore handles POST /library/unmatched/{id}/unignore. +func (h *AdoptionHandler) Unignore(w http.ResponseWriter, r *http.Request) { + h.moveState(w, r, db.UnmatchedStateIgnored, db.UnmatchedStatePending) +} + +func (h *AdoptionHandler) moveState(w http.ResponseWriter, r *http.Request, from, to string) { + id, ok := unitIDParam(w, r) + if !ok { + return + } + moved, err := h.units.ClaimState(r.Context(), id, from, to) + if err != nil { + writeServerError(w, r, err) + return + } + if !moved { + h.stateConflict(w, r, id) + return + } + h.writeUnit(w, r, id) +} + +// maxBulkIgnoreIDs bounds one bulk ignore by id. +const maxBulkIgnoreIDs = 500 + +// IgnoreMany handles POST /library/unmatched/ignore with {ids} or +// {authorFolder}. Only pending rows move. +func (h *AdoptionHandler) IgnoreMany(w http.ResponseWriter, r *http.Request) { + var req struct { + IDs []int64 `json:"ids"` + AuthorFolder string `json:"authorFolder"` + } + if err := decodeAdoptionBody(w, r, 64<<10, &req); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } + req.AuthorFolder = strings.TrimSpace(req.AuthorFolder) + if (len(req.IDs) == 0) == (req.AuthorFolder == "") { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "send either ids or authorFolder"}) + return + } + if len(req.IDs) > maxBulkIgnoreIDs { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "too many ids"}) + return + } + n, err := h.units.IgnorePending(r.Context(), req.IDs, req.AuthorFolder) + if err != nil { + writeServerError(w, r, err) + return + } + writeJSON(w, http.StatusOK, map[string]int64{"ignored": n}) +} + +// decodeAdoptionBody reads a bounded JSON body and refuses any field the +// request type does not declare. An adoption route never takes a path, and a +// client sending one ("path": "/etc/passwd") should learn that it was not +// used rather than have it silently dropped. +func decodeAdoptionBody(w http.ResponseWriter, r *http.Request, limit int64, dst any) error { + dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, limit)) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + if strings.HasPrefix(err.Error(), "json: unknown field ") { + return fmt.Errorf("unknown field %s", strings.TrimPrefix(err.Error(), "json: unknown field ")) + } + return errors.New("invalid request body") + } + return nil +} + +// errAdoptionRefused carries a status and sentence back to the handler. +type errAdoptionRefused struct { + status int + msg string +} + +func (e *errAdoptionRefused) Error() string { return e.msg } + +func refuse(status int, msg string) error { return &errAdoptionRefused{status: status, msg: msg} } + +// writeAdoptionError answers any error an adopt or undo returned. +func (h *AdoptionHandler) writeAdoptionError(w http.ResponseWriter, r *http.Request, err error) { + var refused *errAdoptionRefused + if errors.As(err, &refused) { + writeJSON(w, refused.status, map[string]string{"error": refused.msg}) + return + } + var unavailable *primaryProviderUnavailableError + if errors.As(err, &unavailable) { + writePrimaryProviderUnavailable(w, unavailable.Primary) + return + } + var lookup *addBookLookupError + if errors.As(err, &lookup) { + writeJSON(w, http.StatusBadGateway, map[string]string{"error": lookup.Err.Error()}) + return + } + for _, resp := range addBookErrorResponses { + if errors.Is(err, resp.err) { + writeJSON(w, resp.status, map[string]string{"error": resp.body}) + return + } + } + writeServerError(w, r, err) +} diff --git a/internal/api/adoption_adopt.go b/internal/api/adoption_adopt.go new file mode 100644 index 000000000..23b957f5e --- /dev/null +++ b/internal/api/adoption_adopt.go @@ -0,0 +1,316 @@ +package api + +import ( + "context" + "errors" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/models" +) + +// adoptRequest names the book a unit is, one of two ways: a book already in +// the library (bookId), or a metadata result to add (foreignBookId with its +// author). There is no path field; the unit's files come from its row. +type adoptRequest struct { + BookID int64 `json:"bookId"` + ForeignBookID string `json:"foreignBookId"` + ForeignAuthorID string `json:"foreignAuthorId"` + AuthorName string `json:"authorName"` + // Format, when sent, must be the format of the unit's files (ebook or + // audiobook). Kept for clients that send it; it cannot turn audio files + // into an ebook. + Format string `json:"format"` +} + +// Adopt handles POST /library/unmatched/{id}/adopt. +// +// Adoption is the scanner's own reconcile with a person supplying the match: +// the unit's files are registered in place with the same book_files write the +// scan uses (AddBookFileIfMissing is AddBookFile that also reports whether it +// inserted the row, which is what makes Undo exact). Nothing is moved, renamed +// or queued. +// +// A book added from metadata is created unmonitored and with its media type +// set to the adopted format, so a default of "both" cannot leave the other +// format Wanted and have it grabbed unasked. SkipCatalogueSync is set: the add +// never pulls the author's bibliography anyway (#1816), and skipping the +// single work fallback makes a provider failure fail at once instead of +// polling for 15 seconds, and means no background sync can create the book +// after this request has already given up on it. +// +// Atomicity is by compensation. The row is claimed first (S9); every file is +// checked on disk and against the library roots before anything is written +// (S10); each side effect is written into the row before the next one, so a +// request that dies part way can be reversed by RecoverStaleClaims; and if +// registering fails, what was registered is untracked and a book or author +// this request created is removed again. +func (h *AdoptionHandler) Adopt(w http.ResponseWriter, r *http.Request) { + id, ok := unitIDParam(w, r) + if !ok { + return + } + var req adoptRequest + if err := decodeAdoptionBody(w, r, 16<<10, &req); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } + req.ForeignBookID = strings.TrimSpace(req.ForeignBookID) + if (req.BookID > 0) == (req.ForeignBookID != "") { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "send either bookId or foreignBookId"}) + return + } + switch req.Format { + case "", models.MediaTypeEbook, models.MediaTypeAudiobook: + default: + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "format must be 'ebook' or 'audiobook'"}) + return + } + + ctx := r.Context() + token, err := h.units.Claim(ctx, id, db.UnmatchedStatePending, db.UnmatchedStateAdopting) + if err != nil { + writeServerError(w, r, err) + return + } + if token == "" { + h.stateConflict(w, r, id) + return + } + // The request context may be cancelled once work has started; releasing + // the claim and compensating must still happen. + work := context.WithoutCancel(ctx) + if err := h.adopt(work, id, token, req); err != nil { + var unfinished *unfinishedReversalError + if errors.As(err, &unfinished) { + // The claim and its record stay, so stale claim recovery can finish + // the reversal; clearing them now would forget what to reverse. + slog.Warn("adoption: adopt failed and could not be fully reversed", "unit", id, + "error", unfinished.cause, "reverse_error", unfinished.reverse) + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": unfinishedAdoptMessage}) + return + } + if _, rerr := h.units.ResetToPending(work, id, db.UnmatchedStateAdopting, token); rerr != nil { + slog.Warn("adoption: could not release claim", "unit", id, "error", rerr) + } + h.writeAdoptionError(w, r, err) + return + } + h.writeUnit(w, r, id) +} + +func (h *AdoptionHandler) adopt(ctx context.Context, id int64, token string, req adoptRequest) error { + unit, err := h.units.Get(ctx, id) + if err != nil { + return err + } + if unit == nil { + return refuse(http.StatusNotFound, "unmatched book not found") + } + if req.Format != "" && req.Format != unit.Format { + return refuse(http.StatusBadRequest, "These files are "+unit.Format+" files, so they can only be adopted as "+unit.Format+".") + } + format := unit.Format + paths, err := h.registrationPaths(ctx, unit) + if err != nil { + return err + } + // Refuse before any side effect when a file already belongs to a book. + if err := h.checkOwnership(ctx, append(paths, unit.MemberPaths...), 0); err != nil { + return err + } + + var book *models.Book + var created addBookResult + if req.BookID > 0 { + if book, err = h.books.GetByID(ctx, req.BookID); err != nil { + return err + } + if book == nil { + return refuse(http.StatusNotFound, "That book is no longer in your library.") + } + } else { + if h.adder == nil { + return refuse(http.StatusServiceUnavailable, "Adding books is not available.") + } + unmonitored := false + created, err = h.adder.addBookCore(ctx, addBookParams{ + ForeignBookID: req.ForeignBookID, + ForeignAuthorID: strings.TrimSpace(req.ForeignAuthorID), + AuthorName: strings.TrimSpace(req.AuthorName), + SearchOnAdd: false, + MediaType: format, + Monitored: &unmonitored, + SkipCatalogueSync: true, + }) + var inLibrary *bookInLibraryError + switch { + case errors.As(err, &inLibrary): + // Already in the library: adopt into that row, as if picked. + book = inLibrary.Book + case err != nil: + return err + default: + book = created.Book + } + } + + rec := db.AdoptionRecord{BookID: book.ID} + if created.BookCreated { + rec.CreatedBookID = book.ID + } + if created.AuthorCreated && created.Author != nil { + rec.CreatedAuthorID = created.Author.ID + } + // The created rows are on record before any file is registered. + if err := h.progress(ctx, id, token, rec); err != nil { + return h.failAdopt(ctx, id, rec, err) + } + + regErr := h.register(ctx, id, token, book.ID, format, paths, unit.MemberPaths, &rec) + if regErr == nil && rec.CreatedBookID > 0 { + // A book this request created may also have picked up a file from + // the add's own library lookup. Everything on it came from this + // request, so Undo takes all of it. Then record the book as the + // adoption leaves it, so Undo can tell whether anyone used it since. + files, err := h.books.ListFiles(ctx, book.ID) + if err != nil { + regErr = err + } else { + rec.Registered = rec.Registered[:0] + for _, f := range files { + rec.Registered = append(rec.Registered, db.RegisteredFile{Path: f.Path, BookID: book.ID}) + } + if rec.CreatedBookFingerprint, err = h.units.BookFingerprint(ctx, book.ID); err != nil { + regErr = err + } + } + } + if regErr != nil { + return h.failAdopt(ctx, id, rec, regErr) + } + done, err := h.units.CompleteAdoption(ctx, id, token, rec) + if err != nil || !done { + if err == nil { + err = refuse(http.StatusConflict, "This book changed while it was being adopted. Try again.") + } + return h.failAdopt(ctx, id, rec, err) + } + slog.Info("adoption: registered files in place", "unit", id, "book_id", book.ID, + "files", len(rec.Registered), "book_created", rec.CreatedBookID > 0, "author_created", rec.CreatedAuthorID > 0) + return nil +} + +// failAdopt reverses a failed adopt's writes. If the reversal itself fails +// the result says so, and the caller keeps the claim for recovery. +func (h *AdoptionHandler) failAdopt(ctx context.Context, id int64, rec db.AdoptionRecord, cause error) error { + if _, err := h.reverse(ctx, id, rec); err != nil { + return &unfinishedReversalError{cause: cause, reverse: err} + } + return cause +} + +// progress writes what the adopt has done so far into its row, scoped to this +// request's claim. +func (h *AdoptionHandler) progress(ctx context.Context, id int64, token string, rec db.AdoptionRecord) error { + ok, err := h.units.RecordAdoptionProgress(ctx, id, token, rec) + if err != nil { + return err + } + if !ok { + return refuse(http.StatusConflict, "This book changed while it was being adopted. Try again.") + } + return nil +} + +// registrationPaths decides what goes into book_files for a unit and checks +// every file first. An audiobook folder is registered folder by folder: the +// folder itself for loose tracks, and each disc folder of a disc set. That is +// the shape the scan already treats as tracked (a tracked audio file's folder +// covers its sibling tracks), so no disc rule is needed in the scanner. +// Anything else registers each member file. +// +// Each member must still be a regular file (Lstat, so a symlink swapped in +// since the scan is refused) and must resolve inside a library root: the row +// may be hours old (S10). The path registered is the one the scan walked, so +// the next scan recognises it. +func (h *AdoptionHandler) registrationPaths(ctx context.Context, unit *db.UnmatchedUnit) ([]string, error) { + if len(unit.MemberPaths) == 0 { + return nil, refuse(http.StatusUnprocessableEntity, "This book has no files left. Scan again to refresh the list.") + } + for _, p := range unit.MemberPaths { + info, err := os.Lstat(p) + if err != nil { + return nil, refuse(http.StatusUnprocessableEntity, "A file of this book is gone: "+filepath.Base(p)+". Scan again to refresh the list.") + } + if !info.Mode().IsRegular() { + return nil, refuse(http.StatusUnprocessableEntity, filepath.Base(p)+" is not a regular file, so it cannot be adopted.") + } + if _, ok := h.roots.ResolveContained(ctx, p); !ok { + return nil, refuse(http.StatusUnprocessableEntity, filepath.Base(p)+" is outside your library folders, so it cannot be adopted.") + } + } + if unit.UnitKind != db.UnmatchedKindFolder || unit.Format != models.MediaTypeAudiobook { + out := make([]string, len(unit.MemberPaths)) + for i, p := range unit.MemberPaths { + out[i] = filepath.Clean(p) + } + return out, nil + } + var folders []string + seen := make(map[string]bool) + for _, p := range unit.MemberPaths { + dir := filepath.Clean(filepath.Dir(p)) + if seen[dir] { + continue + } + if _, ok := h.roots.ResolveContained(ctx, dir); !ok { + return nil, refuse(http.StatusUnprocessableEntity, "This folder is outside your library folders, so it cannot be adopted.") + } + seen[dir] = true + folders = append(folders, dir) + } + return folders, nil +} + +// checkOwnership refuses when any path already belongs to a book other than +// bookID (0 means any book at all). +func (h *AdoptionHandler) checkOwnership(ctx context.Context, paths []string, bookID int64) error { + for _, p := range paths { + owned, err := h.books.PathOwnedByOtherBook(ctx, p, bookID) + if err != nil { + return err + } + if owned { + return refuse(http.StatusConflict, filepath.Base(p)+" already belongs to a book in your library.") + } + } + return nil +} + +// register writes the book_files rows, recording each one in rec and in the +// unit's row as soon as it is inserted. On error the caller reverses rec. +func (h *AdoptionHandler) register(ctx context.Context, unitID int64, token string, bookID int64, format string, paths, members []string, rec *db.AdoptionRecord) error { + // Re-check against the resolved book: a concurrent import may have taken + // a file in the moments since the first check. + if err := h.checkOwnership(ctx, append(append([]string{}, paths...), members...), bookID); err != nil { + return err + } + for _, p := range paths { + inserted, err := h.registerFile(ctx, bookID, format, p) + if err != nil { + return err + } + if inserted { + rec.Registered = append(rec.Registered, db.RegisteredFile{Path: p, BookID: bookID}) + if err := h.progress(ctx, unitID, token, *rec); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/api/adoption_recovery_test.go b/internal/api/adoption_recovery_test.go new file mode 100644 index 000000000..de8c9a81e --- /dev/null +++ b/internal/api/adoption_recovery_test.go @@ -0,0 +1,177 @@ +package api + +import ( + "context" + "errors" + "fmt" + "net/http" + "testing" + "time" + + "github.com/vavallee/bindery/internal/db" +) + +// TestRecoverStaleClaims_ReversesADeadAdopt: the request dies (here a panic) +// after registering one file of two. The row still says adopting and records +// what was done; recovery reverses it exactly as Undo would and returns the +// unit to pending, instead of leaving a tracked file and a created book behind +// for the next scan to hide. +func TestRecoverStaleClaims_ReversesADeadAdopt(t *testing.T) { + f := newAdoptionFixture(t, addBookBackCatalogueStub(false)) + ctx := context.Background() + epub := f.write(t, "H. G. Wells/War of the Worlds.epub") + mobi := f.write(t, "H. G. Wells/War of the Worlds.mobi") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: epub, MemberPaths: []string{epub, mobi}}) + + real := f.h.registerFile + calls := 0 + f.h.registerFile = func(ctx context.Context, bookID int64, format, path string) (bool, error) { + calls++ + if calls == 2 { + panic("process killed mid adopt") + } + return real(ctx, bookID, format, path) + } + func() { + defer func() { _ = recover() }() + f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{ + "foreignBookId": "OL27482W", "foreignAuthorId": "OL39307A", "authorName": "H. G. Wells", + }) + }() + + u, _ := f.units.Get(ctx, id) + if u.State != db.UnmatchedStateAdopting || len(u.Registered) != 1 || u.CreatedBookID == 0 || u.CreatedAuthorID == 0 { + t.Fatalf("row after the dead request = %+v, want adopting with one registered file and the created ids", u) + } + bookID, authorID := u.CreatedBookID, u.CreatedAuthorID + + // A fresh claim is not stale yet. + if n, err := f.h.RecoverStaleClaims(ctx, db.UnmatchedClaimTimeout); err != nil || n != 0 { + t.Fatalf("recovery of a fresh claim = %d %v, want 0", n, err) + } + n, err := f.h.RecoverStaleClaims(ctx, -time.Second) + if err != nil || n != 1 { + t.Fatalf("recovery = %d %v, want 1", n, err) + } + if owned, _ := f.books.PathOwnedByOtherBook(ctx, epub, 0); owned { + t.Fatal("registered file still tracked after recovery") + } + if b, _ := f.books.GetByID(ctx, bookID); b != nil { + t.Fatalf("created book left behind: %+v", b) + } + if a, _ := f.authors.GetByID(ctx, authorID); a != nil { + t.Fatalf("created author left behind: %+v", a) + } + if u, _ := f.units.Get(ctx, id); u.State != db.UnmatchedStatePending || len(u.Registered) != 0 { + t.Fatalf("row after recovery = %+v, want pending and clear", u) + } +} + +// refusingUndoStore makes CompleteUndo lose its compare and swap. +type refusingUndoStore struct{ *db.UnmatchedUnitRepo } + +func (refusingUndoStore) CompleteUndo(context.Context, int64, string) (bool, error) { + return false, nil +} + +// TestUndo_ReportsALostCompletion: when the row stopped being held by this +// undo before it finished, the answer is a 409, not a success. +func TestUndo_ReportsALostCompletion(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + book := f.seedBook(t, "Ancillary Mercy") + path := f.write(t, "Ann Leckie/Ancillary Mercy.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": book.ID}); rec.Code != http.StatusOK { + t.Fatalf("adopt = %d", rec.Code) + } + f.h.units = refusingUndoStore{f.units} + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/undo", id), nil); rec.Code != http.StatusConflict { + t.Fatalf("undo with a lost completion = %d %s, want 409", rec.Code, rec.Body.String()) + } +} + +// TestUndo_KeepsTheRecordWhenReversalFails: the database refuses one untrack +// (SQLITE_BUSY in production). Undo must not report success or clear the +// record while a file is still tracked: it answers 503 and keeps the claim, +// and the next recovery pass finishes the job. +func TestUndo_KeepsTheRecordWhenReversalFails(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + ctx := context.Background() + book := f.seedBook(t, "Ancillary Justice") + epub := f.write(t, "Ann Leckie/Ancillary Justice.epub") + mobi := f.write(t, "Ann Leckie/Ancillary Justice.mobi") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: epub, MemberPaths: []string{epub, mobi}}) + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": book.ID}); rec.Code != http.StatusOK { + t.Fatalf("adopt = %d %s", rec.Code, rec.Body.String()) + } + + real := f.h.untrackFile + f.h.untrackFile = func(ctx context.Context, path string, bookID int64) (bool, error) { + if path == mobi { + return false, errors.New("database is locked (SQLITE_BUSY)") + } + return real(ctx, path, bookID) + } + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/undo", id), nil) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("undo with a failing untrack = %d %s, want 503", rec.Code, rec.Body.String()) + } + u, _ := f.units.Get(ctx, id) + if u.State != db.UnmatchedStateUndoing || len(u.Registered) != 2 { + t.Fatalf("row after the failed undo = %+v, want the claim and both registered files on record", u) + } + if got := filePaths(t, f.books, book.ID); len(got) != 1 || got[0] != mobi { + t.Fatalf("book files = %v, want the mobi still tracked", got) + } + + f.h.untrackFile = real + if n, err := f.h.RecoverStaleClaims(ctx, -time.Second); err != nil || n != 1 { + t.Fatalf("recovery = %d %v, want 1", n, err) + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("book files after recovery = %v, want none", got) + } + if u, _ := f.units.Get(ctx, id); u.State != db.UnmatchedStatePending { + t.Fatalf("row after recovery = %s, want pending", u.State) + } +} + +// TestAdopt_KeepsTheClaimWhenItsReversalFails: an adopt fails part way and +// then cannot untrack what it registered. It answers 503 and leaves the row +// claimed with its record, for recovery, instead of returning it to pending +// with a file still tracked and nothing saying so. +func TestAdopt_KeepsTheClaimWhenItsReversalFails(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + ctx := context.Background() + book := f.seedBook(t, "Ancillary Sword") + epub := f.write(t, "Ann Leckie/Ancillary Sword.epub") + mobi := f.write(t, "Ann Leckie/Ancillary Sword.mobi") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: epub, MemberPaths: []string{epub, mobi}}) + + realRegister, realUntrack := f.h.registerFile, f.h.untrackFile + f.h.registerFile = func(ctx context.Context, bookID int64, format, path string) (bool, error) { + if path == mobi { + return false, errors.New("disk on fire") + } + return realRegister(ctx, bookID, format, path) + } + f.h.untrackFile = func(context.Context, string, int64) (bool, error) { + return false, errors.New("database is locked (SQLITE_BUSY)") + } + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": book.ID}) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("adopt = %d %s, want 503", rec.Code, rec.Body.String()) + } + u, _ := f.units.Get(ctx, id) + if u.State != db.UnmatchedStateAdopting || len(u.Registered) != 1 { + t.Fatalf("row = %+v, want the claim kept with the registered file on record", u) + } + + f.h.untrackFile = realUntrack + if n, err := f.h.RecoverStaleClaims(ctx, -time.Second); err != nil || n != 1 { + t.Fatalf("recovery = %d %v", n, err) + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("book files after recovery = %v, want none", got) + } +} diff --git a/internal/api/adoption_test.go b/internal/api/adoption_test.go new file mode 100644 index 000000000..585ca8189 --- /dev/null +++ b/internal/api/adoption_test.go @@ -0,0 +1,517 @@ +package api + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/jobs" + "github.com/vavallee/bindery/internal/metadata" + "github.com/vavallee/bindery/internal/models" +) + +type adoptionFixture struct { + h *AdoptionHandler + router chi.Router + units *db.UnmatchedUnitRepo + books *db.BookRepo + authors *db.AuthorRepo + lib string + db *sql.DB +} + +// newAdoptionFixture wires the handler over an in memory database, a real +// AuthorHandler on the given provider (T1: the stubMetaProvider doubles), and +// a temporary library root. +func newAdoptionFixture(t *testing.T, provider metadata.Provider) adoptionFixture { + t.Helper() + database, err := db.OpenMemory() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + authors := db.NewAuthorRepo(database) + books := db.NewBookRepo(database) + group := jobs.NewGroup(context.Background()) + t.Cleanup(func() { group.Shutdown(5 * time.Second) }) + ah := NewAuthorHandler(authors, nil, books, nil, metadata.NewAggregator(provider), nil, + db.NewMetadataProfileRepo(database), nil).WithJobs(group) + lib := t.TempDir() + if real, err := filepath.EvalSymlinks(lib); err == nil { + lib = real + } + units := db.NewUnmatchedUnitRepo(database) + h := NewAdoptionHandler(units, books, authors, ah, NewLibraryRoots(nil, lib), nil, db.NewSettingsRepo(database)) + r := chi.NewRouter() + r.Get("/library/unmatched", h.List) + r.Get("/library/unmatched/summary", h.Summary) + r.Post("/library/unmatched/ignore", h.IgnoreMany) + r.Post("/library/unmatched/{id}/adopt", h.Adopt) + r.Post("/library/unmatched/{id}/undo", h.Undo) + r.Post("/library/unmatched/{id}/ignore", h.Ignore) + r.Post("/library/unmatched/{id}/unignore", h.Unignore) + return adoptionFixture{h: h, router: r, units: units, books: books, authors: authors, lib: lib, db: database} +} + +func (f adoptionFixture) write(t *testing.T, rel string) string { + t.Helper() + p := filepath.Join(f.lib, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("book"), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +// seedUnit stores one unit the way a scan would and returns its id. +func (f adoptionFixture) seedUnit(t *testing.T, u db.UnmatchedUnitScan) int64 { + t.Helper() + if u.UnitKind == "" { + u.UnitKind = db.UnmatchedKindFile + } + if u.Format == "" { + u.Format = models.MediaTypeEbook + } + u.FileCount = len(u.MemberPaths) + u.RootPath = f.lib + if _, err := f.units.ReconcileScan(context.Background(), []db.UnmatchedUnitScan{u}, db.ReconcileScanOptions{SkipDeletion: true}); err != nil { + t.Fatal(err) + } + items, _, err := f.units.List(context.Background(), db.UnmatchedListQuery{Limit: 250}) + if err != nil { + t.Fatal(err) + } + for _, it := range items { + if it.UnitPath == u.UnitPath { + return it.ID + } + } + t.Fatalf("seeded unit %s not listed", u.UnitPath) + return 0 +} + +func (f adoptionFixture) post(t *testing.T, path string, body any) *httptest.ResponseRecorder { + t.Helper() + var buf bytes.Buffer + if body != nil { + if err := json.NewEncoder(&buf).Encode(body); err != nil { + t.Fatal(err) + } + } + rec := httptest.NewRecorder() + f.router.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, path, &buf)) + return rec +} + +func (f adoptionFixture) seedBook(t *testing.T, title string) *models.Book { + t.Helper() + ctx := context.Background() + author := &models.Author{ForeignID: "ol:" + title, Name: "Ann Leckie", SortName: "Leckie, Ann", MetadataProvider: "openlibrary"} + if err := f.authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + b := &models.Book{ForeignID: "ol:b:" + title, AuthorID: author.ID, Title: title, Status: models.BookStatusWanted, + Monitored: true, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := f.books.Create(ctx, b); err != nil { + t.Fatal(err) + } + return b +} + +func decodeItem(t *testing.T, rec *httptest.ResponseRecorder) adoptionItem { + t.Helper() + var it adoptionItem + if err := json.Unmarshal(rec.Body.Bytes(), &it); err != nil { + t.Fatalf("decode %q: %v", rec.Body.String(), err) + } + return it +} + +func filePaths(t *testing.T, books *db.BookRepo, bookID int64) []string { + t.Helper() + files, err := books.ListFiles(context.Background(), bookID) + if err != nil { + t.Fatal(err) + } + out := make([]string, 0, len(files)) + for _, f := range files { + out = append(out, f.Path) + } + return out +} + +// TestAdopt_ExistingBookRegistersInPlaceAndUndoIsExact: adopting into a book +// already in the library registers the file where it is, and undo removes +// exactly that registration and leaves the book. +func TestAdopt_ExistingBookRegistersInPlaceAndUndoIsExact(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + book := f.seedBook(t, "Ancillary Justice") + path := f.write(t, "Ann Leckie/Ancillary Justice.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": book.ID}) + if rec.Code != http.StatusOK { + t.Fatalf("adopt = %d %s", rec.Code, rec.Body.String()) + } + it := decodeItem(t, rec) + if it.State != db.UnmatchedStateAdopted || it.Book == nil || it.Book.ID != book.ID || it.BookCreated || it.AuthorCreated { + t.Fatalf("adopted item = %+v", it) + } + if got := filePaths(t, f.books, book.ID); len(got) != 1 || got[0] != path { + t.Fatalf("book files = %v, want the file in place", got) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("file moved or removed: %v", err) + } + after, _ := f.books.GetByID(context.Background(), book.ID) + if after.Status != models.BookStatusImported || !after.Monitored { + t.Fatalf("book after adopt = status %s monitored %v, want imported and monitoring untouched", after.Status, after.Monitored) + } + + rec = f.post(t, fmt.Sprintf("/library/unmatched/%d/undo", id), nil) + if rec.Code != http.StatusOK || decodeItem(t, rec).State != db.UnmatchedStatePending { + t.Fatalf("undo = %d %s", rec.Code, rec.Body.String()) + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("book files after undo = %v, want none", got) + } + if b, _ := f.books.GetByID(context.Background(), book.ID); b == nil || b.Status != models.BookStatusWanted { + t.Fatalf("existing book after undo = %+v, want kept and wanted again", b) + } +} + +// TestAdopt_CreatesUnmonitoredBookInTheAdoptedFormat: a metadata result is +// added unmonitored with media type set to the unit's format, so nothing is +// left Wanted to be grabbed. An audiobook folder registers as its folder. +// Undo removes the created book and author again. +func TestAdopt_CreatesUnmonitoredBookInTheAdoptedFormat(t *testing.T) { + f := newAdoptionFixture(t, addBookBackCatalogueStub(false)) + var tracks []string + for i := 1; i <= 3; i++ { + tracks = append(tracks, f.write(t, fmt.Sprintf("H. G. Wells/War of the Worlds/%02d.mp3", i))) + } + folder := filepath.Dir(tracks[0]) + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: folder, UnitKind: db.UnmatchedKindFolder, + Format: models.MediaTypeAudiobook, MemberPaths: tracks}) + + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{ + "foreignBookId": "OL27482W", "foreignAuthorId": "OL39307A", "authorName": "H. G. Wells", + }) + if rec.Code != http.StatusOK { + t.Fatalf("adopt = %d %s", rec.Code, rec.Body.String()) + } + it := decodeItem(t, rec) + if !it.BookCreated || !it.AuthorCreated || it.Book == nil { + t.Fatalf("item = %+v, want a created book and author", it) + } + book, _ := f.books.GetByID(context.Background(), it.Book.ID) + if book.Monitored || book.MediaType != models.MediaTypeAudiobook || book.Status != models.BookStatusImported { + t.Fatalf("created book = monitored %v, media %s, status %s; want unmonitored audiobook, imported", + book.Monitored, book.MediaType, book.Status) + } + if got := filePaths(t, f.books, book.ID); len(got) != 1 || got[0] != folder { + t.Fatalf("registered = %v, want the folder %s", got, folder) + } + + authorID := book.AuthorID + rec = f.post(t, fmt.Sprintf("/library/unmatched/%d/undo", id), nil) + if rec.Code != http.StatusOK { + t.Fatalf("undo = %d %s", rec.Code, rec.Body.String()) + } + if b, _ := f.books.GetByID(context.Background(), book.ID); b != nil { + t.Fatalf("created book survived undo: %+v", b) + } + if a, _ := f.authors.GetByID(context.Background(), authorID); a != nil { + t.Fatalf("created author survived undo: %+v", a) + } +} + +// TestAdopt_InjectedFailureStrandsNothing: registration failing part way +// leaves no file row, no created book or author, and the row pending. +func TestAdopt_InjectedFailureStrandsNothing(t *testing.T) { + f := newAdoptionFixture(t, addBookBackCatalogueStub(false)) + epub := f.write(t, "H. G. Wells/War of the Worlds.epub") + mobi := f.write(t, "H. G. Wells/War of the Worlds.mobi") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: epub, MemberPaths: []string{epub, mobi}}) + + real := f.h.registerFile + calls := 0 + f.h.registerFile = func(ctx context.Context, bookID int64, format, path string) (bool, error) { + calls++ + if calls == 2 { + return false, errors.New("disk on fire") + } + return real(ctx, bookID, format, path) + } + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{ + "foreignBookId": "OL27482W", "foreignAuthorId": "OL39307A", "authorName": "H. G. Wells", + }) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("adopt = %d %s, want 500", rec.Code, rec.Body.String()) + } + ctx := context.Background() + if b, _ := f.books.GetByForeignID(ctx, "OL27482W"); b != nil { + t.Fatalf("created book stranded: %+v", b) + } + if a, _ := f.authors.GetByForeignID(ctx, "OL39307A"); a != nil { + t.Fatalf("created author stranded: %+v", a) + } + for _, p := range []string{epub, mobi} { + if owned, _ := f.books.PathOwnedByOtherBook(ctx, p, 0); owned { + t.Fatalf("%s still registered", p) + } + } + if u, _ := f.units.Get(ctx, id); u.State != db.UnmatchedStatePending { + t.Fatalf("unit state = %s, want pending", u.State) + } +} + +// TestAdopt_RefusesAFileAnotherBookOwns is the 409. +func TestAdopt_RefusesAFileAnotherBookOwns(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + owner := f.seedBook(t, "Provenance") + target := f.seedBook(t, "Translation State") + path := f.write(t, "Ann Leckie/Provenance.epub") + if err := f.books.AddBookFile(context.Background(), owner.ID, models.MediaTypeEbook, path); err != nil { + t.Fatal(err) + } + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": target.ID}) + if rec.Code != http.StatusConflict { + t.Fatalf("adopt = %d %s, want 409", rec.Code, rec.Body.String()) + } + if got := filePaths(t, f.books, target.ID); len(got) != 0 { + t.Fatalf("target gained %v", got) + } +} + +// TestAdopt_RejectsSymlinkEscape is S10 at adopt time: the row may be hours +// old, so a member swapped for a symlink, or reached through a symlinked +// folder that leaves the library, is refused and nothing is written. +func TestAdopt_RejectsSymlinkEscape(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + book := f.seedBook(t, "Ancillary Sword") + outside := t.TempDir() + secret := filepath.Join(outside, "secret.epub") + if err := os.WriteFile(secret, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + + swapped := f.write(t, "Ann Leckie/Swapped.epub") + if err := os.Remove(swapped); err != nil { + t.Fatal(err) + } + if err := os.Symlink(secret, swapped); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + escapeDir := filepath.Join(f.lib, "Escape") + if err := os.Symlink(outside, escapeDir); err != nil { + t.Fatal(err) + } + throughDir := filepath.Join(escapeDir, "secret.epub") + + for _, p := range []string{swapped, throughDir} { + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: p, MemberPaths: []string{p}}) + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": book.ID}) + if rec.Code != http.StatusUnprocessableEntity { + t.Errorf("adopt %s = %d %s, want 422", p, rec.Code, rec.Body.String()) + } + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("registered through a symlink: %v", got) + } +} + +// TestAdopt_DoubleAdoptAndDoubleUndo is T3: concurrent requests for one row, +// exactly one wins each time. +func TestAdopt_DoubleAdoptAndDoubleUndo(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + book := f.seedBook(t, "Ancillary Mercy") + path := f.write(t, "Ann Leckie/Ancillary Mercy.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + + race := func(action string, body any) (ok, conflict int32) { + var wg sync.WaitGroup + for range 6 { + wg.Add(1) + go func() { + defer wg.Done() + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/%s", id, action), body) + switch rec.Code { + case http.StatusOK: + atomic.AddInt32(&ok, 1) + case http.StatusConflict: + atomic.AddInt32(&conflict, 1) + default: + t.Errorf("%s = %d %s", action, rec.Code, rec.Body.String()) + } + }() + } + wg.Wait() + return ok, conflict + } + if ok, conflict := race("adopt", map[string]any{"bookId": book.ID}); ok != 1 || conflict != 5 { + t.Fatalf("adopt race: %d ok, %d conflict; want 1 and 5", ok, conflict) + } + if got := filePaths(t, f.books, book.ID); len(got) != 1 { + t.Fatalf("files after adopt race = %v", got) + } + if ok, conflict := race("undo", nil); ok != 1 || conflict != 5 { + t.Fatalf("undo race: %d ok, %d conflict; want 1 and 5", ok, conflict) + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("files after undo race = %v", got) + } +} + +// TestAdopt_BadRequests: no path field exists, and the body must name one +// target. +func TestAdopt_BadRequests(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + path := f.write(t, "X/y.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + for name, body := range map[string]any{ + "neither": map[string]any{}, + "both": map[string]any{"bookId": 1, "foreignBookId": "OL1W"}, + "bad format": map[string]any{"bookId": 1, "format": "both"}, + } { + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), body); rec.Code != http.StatusBadRequest { + t.Errorf("%s: %d, want 400", name, rec.Code) + } + } + if rec := f.post(t, "/library/unmatched/999/adopt", map[string]any{"bookId": 1}); rec.Code != http.StatusConflict && rec.Code != http.StatusNotFound { + t.Errorf("unknown id: %d", rec.Code) + } +} + +// TestIgnore_SingleBulkAndUnignore covers the ignore routes and their state +// guards. +func TestIgnore_SingleBulkAndUnignore(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + a := f.write(t, "Folder A/one.epub") + b := f.write(t, "Folder A/two.epub") + c := f.write(t, "Folder B/three.epub") + ida := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: a, MemberPaths: []string{a}, AuthorFolder: "Folder A"}) + f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: b, MemberPaths: []string{b}, AuthorFolder: "Folder A"}) + idc := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: c, MemberPaths: []string{c}, AuthorFolder: "Folder B"}) + + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/ignore", idc), nil); rec.Code != http.StatusOK { + t.Fatalf("ignore = %d", rec.Code) + } + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/ignore", idc), nil); rec.Code != http.StatusConflict { + t.Fatalf("second ignore = %d, want 409", rec.Code) + } + rec := f.post(t, "/library/unmatched/ignore", map[string]any{"authorFolder": "Folder A"}) + var bulk map[string]int64 + _ = json.Unmarshal(rec.Body.Bytes(), &bulk) + if rec.Code != http.StatusOK || bulk["ignored"] != 2 { + t.Fatalf("bulk ignore = %d %s", rec.Code, rec.Body.String()) + } + if rec := f.post(t, "/library/unmatched/ignore", map[string]any{"ids": []int64{ida}, "authorFolder": "x"}); rec.Code != http.StatusBadRequest { + t.Fatalf("bulk with both selectors = %d, want 400", rec.Code) + } + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/unignore", ida), nil); rec.Code != http.StatusOK || decodeItem(t, rec).State != db.UnmatchedStatePending { + t.Fatalf("unignore = %d %s", rec.Code, rec.Body.String()) + } +} + +// countingUnitStore counts the book hydration queries a page costs. +type countingUnitStore struct { + *db.UnmatchedUnitRepo + bookRefs atomic.Int32 +} + +func (c *countingUnitStore) BookRefs(ctx context.Context, ids []int64) (map[int64]db.UnmatchedBookRef, error) { + c.bookRefs.Add(1) + return c.UnmatchedUnitRepo.BookRefs(ctx, ids) +} + +// countingProvider counts every metadata provider call. +type countingProvider struct { + metadata.Provider + calls atomic.Int32 +} + +func (p *countingProvider) SearchAuthors(ctx context.Context, q string) ([]models.Author, error) { + p.calls.Add(1) + return p.Provider.SearchAuthors(ctx, q) +} +func (p *countingProvider) SearchBooks(ctx context.Context, q string) ([]models.Book, error) { + p.calls.Add(1) + return p.Provider.SearchBooks(ctx, q) +} +func (p *countingProvider) GetAuthor(ctx context.Context, id string) (*models.Author, error) { + p.calls.Add(1) + return p.Provider.GetAuthor(ctx, id) +} +func (p *countingProvider) GetBook(ctx context.Context, id string) (*models.Book, error) { + p.calls.Add(1) + return p.Provider.GetBook(ctx, id) +} +func (p *countingProvider) GetEditions(ctx context.Context, id string) ([]models.Edition, error) { + p.calls.Add(1) + return p.Provider.GetEditions(ctx, id) +} +func (p *countingProvider) GetBookByISBN(ctx context.Context, isbn string) (*models.Book, error) { + p.calls.Add(1) + return p.Provider.GetBookByISBN(ctx, isbn) +} + +// TestAdoptionList_NoProviderCallsAndOneHydrationQuery is T5: a page with +// several rows, each with suggestions, costs one book query and no provider +// call, with and without facets. +func TestAdoptionList_NoProviderCallsAndOneHydrationQuery(t *testing.T) { + provider := &countingProvider{Provider: &stubMetaProvider{name: "openlibrary"}} + f := newAdoptionFixture(t, provider) + counting := &countingUnitStore{UnmatchedUnitRepo: f.units} + f.h.units = counting + + b1 := f.seedBook(t, "Ancillary Justice") + b2 := f.seedBook(t, "Ancillary Sword") + for i := range 5 { + p := f.write(t, fmt.Sprintf("Ann Leckie/Book %d.epub", i)) + f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: p, MemberPaths: []string{p}, AuthorFolder: "Ann Leckie", ParsedTitle: fmt.Sprintf("Book %d", i), + Candidates: []db.UnmatchedCandidate{{BookID: b1.ID, Score: 0.7}, {BookID: b2.ID, Score: 0.65}}}) + } + + for _, q := range []string{"", "?facets=1", "?facets=1&search=book&sort=title"} { + counting.bookRefs.Store(0) + rec := httptest.NewRecorder() + f.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/library/unmatched"+q, nil)) + if rec.Code != http.StatusOK { + t.Fatalf("list%s = %d %s", q, rec.Code, rec.Body.String()) + } + var resp adoptionListResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Total != 5 || len(resp.Items) != 5 || len(resp.Items[0].Candidates) != 2 || resp.Items[0].Candidates[0].Book.Title != "Ancillary Justice" { + t.Fatalf("list%s = %+v", q, resp) + } + if n := counting.bookRefs.Load(); n != 1 { + t.Errorf("list%s hydration queries = %d, want 1", q, n) + } + if (q != "") != (resp.Facets != nil) { + t.Errorf("list%s facets present = %v", q, resp.Facets != nil) + } + } + if n := provider.calls.Load(); n != 0 { + t.Fatalf("provider calls during list = %d, want 0", n) + } +} diff --git a/internal/api/adoption_undo.go b/internal/api/adoption_undo.go new file mode 100644 index 000000000..b28a7ea17 --- /dev/null +++ b/internal/api/adoption_undo.go @@ -0,0 +1,208 @@ +package api + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "time" + + "github.com/vavallee/bindery/internal/db" +) + +// Undo, the compensation a failed adopt runs, and the recovery of claims a +// dead request left behind all reverse an adoption the same way: reverse. + +// keptBookMessage is what Undo says when the created book stays. +const keptBookMessage = "The files are no longer adopted. The book it added stays in your library because it has been used since." + +// Answers when a reversal could not finish. The claim and the record stay, so +// stale claim recovery completes the reversal later. +const ( + unfinishedUndoMessage = "Undo could not finish, most likely because the database was busy. Nothing is lost: it will be completed automatically within a few minutes, or try again then." + unfinishedAdoptMessage = "Adopting failed and could not be fully reversed yet. It will be cleaned up automatically within a few minutes." +) + +// unfinishedReversalError is a failed adopt whose reversal also failed. +type unfinishedReversalError struct { + cause, reverse error +} + +func (e *unfinishedReversalError) Error() string { + return fmt.Sprintf("%v (reversal unfinished: %v)", e.cause, e.reverse) +} + +func (e *unfinishedReversalError) Unwrap() error { return e.cause } + +// Undo handles POST /library/unmatched/{id}/undo. It untracks exactly the +// book_files rows the adoption registered, each only while it still belongs +// to the book it was registered to, then removes a book or author the +// adoption created when nothing else holds it and nobody has used it since +// (S9). The unit returns to pending only once all of that succeeded; if any +// step fails the row keeps its claim and record and the answer is 503, so +// nothing about the adoption is forgotten while files may still be tracked. +func (h *AdoptionHandler) Undo(w http.ResponseWriter, r *http.Request) { + id, ok := unitIDParam(w, r) + if !ok { + return + } + ctx := r.Context() + token, err := h.units.Claim(ctx, id, db.UnmatchedStateAdopted, db.UnmatchedStateUndoing) + if err != nil { + writeServerError(w, r, err) + return + } + if token == "" { + h.stateConflict(w, r, id) + return + } + work := context.WithoutCancel(ctx) + unit, err := h.units.Get(work, id) + if err != nil || unit == nil { + if _, rerr := h.units.ReleaseClaim(work, id, db.UnmatchedStateUndoing, db.UnmatchedStateAdopted, token); rerr != nil { + slog.Warn("adoption: could not release undo claim", "unit", id, "error", rerr) + } + if err == nil { + err = refuse(http.StatusNotFound, "unmatched book not found") + } + h.writeAdoptionError(w, r, err) + return + } + kept, err := h.reverse(work, id, recordOf(unit)) + if err != nil { + slog.Warn("adoption: undo could not finish; left for recovery", "unit", id, "error", err) + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": unfinishedUndoMessage}) + return + } + done, err := h.units.CompleteUndo(work, id, token) + if err != nil { + writeServerError(w, r, err) + return + } + if !done { + h.stateConflict(w, r, id) + return + } + slog.Info("adoption: undone", "unit", id, "book_id", unit.BookID, "files", len(unit.Registered), "kept_created_book", kept) + message := "" + if kept { + message = keptBookMessage + } + h.writeUnitWithMessage(w, r, id, message) +} + +func recordOf(u *db.UnmatchedUnit) db.AdoptionRecord { + return db.AdoptionRecord{ + BookID: u.BookID, CreatedBookID: u.CreatedBookID, CreatedAuthorID: u.CreatedAuthorID, + CreatedBookFingerprint: u.CreatedBookFingerprint, Registered: u.Registered, + } +} + +// reverse undoes an adoption's writes and reports whether a created book was +// kept because it has been used since. Every step is idempotent, so a reversal +// that fails part way can simply run again; the first error stops it and is +// returned, and the caller must then keep the record. +// +// The fingerprint is compared before untracking, because untracking itself +// updates the book. +func (h *AdoptionHandler) reverse(ctx context.Context, unitID int64, rec db.AdoptionRecord) (bool, error) { + used := false + if rec.CreatedBookID > 0 && rec.CreatedBookFingerprint != "" { + now, err := h.units.BookFingerprint(ctx, rec.CreatedBookID) + if err != nil { + return false, fmt.Errorf("fingerprint created book %d: %w", rec.CreatedBookID, err) + } + used = now != "" && now != rec.CreatedBookFingerprint + } + for _, f := range rec.Registered { + if _, err := h.untrackFile(ctx, f.Path, f.BookID); err != nil { + return false, fmt.Errorf("untrack %s: %w", f.Path, err) + } + } + if used { + return true, nil + } + if rec.CreatedBookID > 0 { + ours, err := h.bookIsOnlyOurs(ctx, unitID, rec.CreatedBookID) + if err != nil { + return false, err + } + if ours { + if err := h.books.Delete(ctx, rec.CreatedBookID); err != nil { + return false, fmt.Errorf("remove created book %d: %w", rec.CreatedBookID, err) + } + } + } + if rec.CreatedAuthorID > 0 { + // Including excluded books: deleting the author cascades to them. + books, err := h.books.ListByAuthorIncludingExcluded(ctx, rec.CreatedAuthorID) + if err != nil { + return false, fmt.Errorf("list books of created author %d: %w", rec.CreatedAuthorID, err) + } + if len(books) > 0 { + return false, nil + } + other, err := h.units.AuthorReferencedElsewhere(ctx, rec.CreatedAuthorID, unitID) + if err != nil { + return false, err + } + if other { + return false, nil + } + if err := h.authors.Delete(ctx, rec.CreatedAuthorID); err != nil { + return false, fmt.Errorf("remove created author %d: %w", rec.CreatedAuthorID, err) + } + } + return false, nil +} + +func (h *AdoptionHandler) bookIsOnlyOurs(ctx context.Context, unitID, bookID int64) (bool, error) { + files, err := h.books.ListFiles(ctx, bookID) + if err != nil { + return false, fmt.Errorf("list files of created book %d: %w", bookID, err) + } + if len(files) > 0 { + return false, nil + } + other, err := h.units.BookReferencedElsewhere(ctx, bookID, unitID) + if err != nil { + return false, err + } + return !other, nil +} + +// RecoverStaleClaims reverses and releases rows an adopt or undo claimed +// longer than olderThan ago: requests that died holding them. The row records +// every side effect before the next one, so the reversal is the same one Undo +// runs, with the same limits. A reversal that fails leaves the row claimed for +// the next pass. Every write is scoped to the stale claim's token, so a row +// another request has claimed since is left alone. Call it with zero at +// startup, when any claim is from a previous process, and with +// db.UnmatchedClaimTimeout later. +func (h *AdoptionHandler) RecoverStaleClaims(ctx context.Context, olderThan time.Duration) (int, error) { + stale, err := h.units.StaleClaims(ctx, time.Now().Add(-olderThan)) + if err != nil { + return 0, err + } + recovered := 0 + var errs []error + for i := range stale { + u := &stale[i] + if _, err := h.reverse(ctx, u.ID, recordOf(u)); err != nil { + slog.Warn("adoption: could not reverse an abandoned claim; will retry", "unit", u.ID, "error", err) + errs = append(errs, err) + continue + } + ok, err := h.units.ResetToPending(ctx, u.ID, u.State, u.ClaimToken) + if err != nil { + errs = append(errs, err) + continue + } + if ok { + recovered++ + slog.Warn("adoption: recovered an abandoned claim", "unit", u.ID, "state", u.State, "files", len(u.Registered)) + } + } + return recovered, errors.Join(errs...) +} diff --git a/internal/api/adoption_undo_safety_test.go b/internal/api/adoption_undo_safety_test.go new file mode 100644 index 000000000..a1ee769c7 --- /dev/null +++ b/internal/api/adoption_undo_safety_test.go @@ -0,0 +1,230 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "path/filepath" + "strings" + "testing" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/models" +) + +// These tests come from the backend review of library adoption. Each one is a +// way Undo or adopt could lose data that was not the adoption's to lose. + +func (f adoptionFixture) adoptWells(t *testing.T, id int64) adoptionItem { + t.Helper() + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{ + "foreignBookId": "OL27482W", "foreignAuthorId": "OL39307A", "authorName": "H. G. Wells", + }) + if rec.Code != http.StatusOK { + t.Fatalf("adopt = %d %s", rec.Code, rec.Body.String()) + } + it := decodeItem(t, rec) + if !it.BookCreated || !it.AuthorCreated || it.Book == nil { + t.Fatalf("adopt did not create book and author: %+v", it) + } + return it +} + +// TestUndo_LeavesAFileThatNowBelongsToAnotherBook: adopt into B, delete B, +// register the same path to C, Undo. C keeps its file. +func TestUndo_LeavesAFileThatNowBelongsToAnotherBook(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + ctx := context.Background() + b := f.seedBook(t, "Ancillary Justice") + c := f.seedBook(t, "Ancillary Sword") + path := f.write(t, "Ann Leckie/Ancillary.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": b.ID}); rec.Code != http.StatusOK { + t.Fatalf("adopt = %d %s", rec.Code, rec.Body.String()) + } + if err := f.books.Delete(ctx, b.ID); err != nil { + t.Fatal(err) + } + if err := f.books.AddBookFile(ctx, c.ID, models.MediaTypeEbook, path); err != nil { + t.Fatal(err) + } + + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/undo", id), nil) + if rec.Code != http.StatusOK { + t.Fatalf("undo = %d %s", rec.Code, rec.Body.String()) + } + if got := filePaths(t, f.books, c.ID); len(got) != 1 || got[0] != path { + t.Fatalf("book C files after undo = %v, want it to keep %s", got, path) + } + if u, _ := f.units.Get(ctx, id); u.State != db.UnmatchedStatePending { + t.Fatalf("unit state = %s, want pending", u.State) + } +} + +// TestUndo_KeepsACreatedAuthorWithAnExcludedBook: the created author gains an +// excluded book with a file. Undo must not delete the author, which would +// cascade to that book. +func TestUndo_KeepsACreatedAuthorWithAnExcludedBook(t *testing.T) { + f := newAdoptionFixture(t, addBookBackCatalogueStub(false)) + ctx := context.Background() + path := f.write(t, "H. G. Wells/War of the Worlds.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + it := f.adoptWells(t, id) + adopted, _ := f.books.GetByID(ctx, it.Book.ID) + + other := &models.Book{ForeignID: "OL-excluded", AuthorID: adopted.AuthorID, Title: "The Time Machine", + Status: models.BookStatusWanted, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := f.books.Create(ctx, other); err != nil { + t.Fatal(err) + } + otherPath := f.write(t, "H. G. Wells/The Time Machine.epub") + if err := f.books.AddBookFile(ctx, other.ID, models.MediaTypeEbook, otherPath); err != nil { + t.Fatal(err) + } + if err := f.books.SetExcluded(ctx, other.ID, true); err != nil { + t.Fatal(err) + } + + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/undo", id), nil); rec.Code != http.StatusOK { + t.Fatalf("undo = %d %s", rec.Code, rec.Body.String()) + } + if a, _ := f.authors.GetByID(ctx, adopted.AuthorID); a == nil { + t.Fatal("created author deleted although it still has an excluded book") + } + if b, _ := f.books.GetByID(ctx, other.ID); b == nil { + t.Fatal("excluded book deleted with the author") + } + if got := filePaths(t, f.books, other.ID); len(got) != 1 { + t.Fatalf("excluded book files = %v, want its file kept", got) + } +} + +// TestUndo_KeepsACreatedBookSomeoneStartedUsing: after the adoption the book +// was monitored, or linked into a series. Undo untracks the adopted files and +// returns the unit to pending, but keeps the book and author, and says so. +func TestUndo_KeepsACreatedBookSomeoneStartedUsing(t *testing.T) { + for _, tc := range []struct { + name string + use func(t *testing.T, f adoptionFixture, book *models.Book) + }{ + {"monitored", func(t *testing.T, f adoptionFixture, book *models.Book) { + book.Monitored = true + if err := f.books.Update(context.Background(), book); err != nil { + t.Fatal(err) + } + }}, + {"blocklist entry", func(t *testing.T, f adoptionFixture, book *models.Book) { + if _, err := f.db.Exec(`INSERT INTO blocklist (book_id, guid, title) VALUES (?, 'guid-1', 'War of the Worlds retail')`, book.ID); err != nil { + t.Fatal(err) + } + }}, + {"calibre id", func(t *testing.T, f adoptionFixture, book *models.Book) { + // Written the way the Calibre integration writes it: the column + // alone, without touching updated_at. + if _, err := f.db.Exec(`UPDATE books SET calibre_id = 42 WHERE id = ?`, book.ID); err != nil { + t.Fatal(err) + } + }}, + {"series link", func(t *testing.T, f adoptionFixture, book *models.Book) { + series := db.NewSeriesRepo(f.db) + s := &models.Series{ForeignID: "OL-series", Title: "Scientific Romances"} + if err := series.CreateOrGet(context.Background(), s); err != nil { + t.Fatal(err) + } + if err := series.LinkBook(context.Background(), s.ID, book.ID, "1", true); err != nil { + t.Fatal(err) + } + }}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newAdoptionFixture(t, addBookBackCatalogueStub(false)) + ctx := context.Background() + path := f.write(t, "H. G. Wells/War of the Worlds.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + it := f.adoptWells(t, id) + book, _ := f.books.GetByID(ctx, it.Book.ID) + tc.use(t, f, book) + + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/undo", id), nil) + if rec.Code != http.StatusOK { + t.Fatalf("undo = %d %s", rec.Code, rec.Body.String()) + } + var body map[string]any + _ = json.Unmarshal(rec.Body.Bytes(), &body) + if kept, _ := f.books.GetByID(ctx, book.ID); kept == nil { + t.Fatal("created book deleted although it was in use") + } + if a, _ := f.authors.GetByID(ctx, book.AuthorID); a == nil { + t.Fatal("created author deleted although its book was kept") + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("adopted files still tracked: %v", got) + } + if msg, _ := body["message"].(string); msg == "" { + t.Errorf("undo response has no message saying the book was kept: %s", rec.Body.String()) + } + }) + } +} + +// TestAdopt_RejectsAFormatThatContradictsTheFiles: audio files cannot be +// adopted as an ebook, nor ebooks as an audiobook. +func TestAdopt_RejectsAFormatThatContradictsTheFiles(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + book := f.seedBook(t, "Provenance") + epub := f.write(t, "Ann Leckie/Provenance.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: epub, MemberPaths: []string{epub}}) + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": book.ID, "format": "audiobook"}) + if rec.Code != http.StatusBadRequest { + t.Fatalf("ebook adopted as audiobook = %d %s, want 400", rec.Code, rec.Body.String()) + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("registered %v", got) + } +} + +// TestAdopt_RegistersEachDiscFolder: a disc set is registered disc folder by +// disc folder, the shape the scan already counts as tracked, so no scanner +// rule about disc folders under a tracked folder is needed. +func TestAdopt_RegistersEachDiscFolder(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + book := f.seedBook(t, "Artemis") + cd1 := f.write(t, "Andy Weir/Artemis/CD1/01.mp3") + cd2 := f.write(t, "Andy Weir/Artemis/CD2/01.mp3") + folder := filepath.Dir(filepath.Dir(cd1)) + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: folder, UnitKind: db.UnmatchedKindFolder, + Format: models.MediaTypeAudiobook, MemberPaths: []string{cd1, cd2}}) + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": book.ID}); rec.Code != http.StatusOK { + t.Fatalf("adopt = %d %s", rec.Code, rec.Body.String()) + } + got := filePaths(t, f.books, book.ID) + if len(got) != 2 || got[0] != filepath.Dir(cd1) || got[1] != filepath.Dir(cd2) { + t.Fatalf("registered %v, want the two disc folders", got) + } +} + +// TestAdoptionBodies_RejectUnknownFields: no adoption route takes a path, and +// one sent anyway is refused with 400 instead of being silently ignored. +func TestAdoptionBodies_RejectUnknownFields(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + book := f.seedBook(t, "Provenance") + epub := f.write(t, "Ann Leckie/Provenance.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: epub, MemberPaths: []string{epub}, AuthorFolder: "Ann Leckie"}) + + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": book.ID, "path": "/etc/passwd"}) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "path") { + t.Fatalf("adopt with a path field = %d %s, want 400 naming the field", rec.Code, rec.Body.String()) + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("registered %v", got) + } + rec = f.post(t, "/library/unmatched/ignore", map[string]any{"authorFolder": "Ann Leckie", "path": "/etc"}) + if rec.Code != http.StatusBadRequest { + t.Fatalf("bulk ignore with a path field = %d %s, want 400", rec.Code, rec.Body.String()) + } + if u, _ := f.units.Get(context.Background(), id); u.State != db.UnmatchedStatePending { + t.Fatalf("unit state = %s, want still pending", u.State) + } +} diff --git a/internal/api/authors.go b/internal/api/authors.go index 01895ace5..86813892f 100644 --- a/internal/api/authors.go +++ b/internal/api/authors.go @@ -25,7 +25,6 @@ import ( "github.com/vavallee/bindery/internal/jobs" "github.com/vavallee/bindery/internal/metadata" "github.com/vavallee/bindery/internal/models" - "github.com/vavallee/bindery/internal/telemetry" "github.com/vavallee/bindery/internal/textutil" ) @@ -520,128 +519,29 @@ func (h *AuthorHandler) Create(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) return } - if req.ForeignID == "" || req.Name == "" { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "foreignAuthorId and authorName required"}) - return - } - monitorMode, monitorLatestCount, err := h.resolveCreateMonitorOptions(r.Context(), req.MonitorMode, req.MonitorLatestCount) - monitorNewItems := models.DefaultAuthorMonitorNewItems - if req.MonitorNewItems != nil { - v := strings.TrimSpace(*req.MonitorNewItems) - if !models.IsAuthorMonitorNewItemsValid(v) { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid monitorNewItems"}) - return - } - monitorNewItems = v - } - if err != nil { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) - return - } - - // Check if already exists — use user-scoped lookup so this agrees with the - // author list, which filters by owner_user_id. A global GetByForeignID - // would block re-creation of authors orphaned under a different user ID. - userID := auth.UserIDFromContext(r.Context()) - existing, _ := h.authors.GetByAnyForeignIDForUser(r.Context(), req.ForeignID, userID) - if existing != nil { - h.writeCanonicalAuthorConflict(w, existing, "author already exists") - return - } - - author, err := h.fetchAuthorForCreate(r.Context(), req.ForeignID, req.Name) + res, err := h.createAuthorCore(r.Context(), createAuthorParams{ + ForeignID: req.ForeignID, + Name: req.Name, + QualityProfileID: req.QualityProfileID, + MetadataProfileID: req.MetadataProfileID, + RootFolderID: req.RootFolderID, + AudiobookRootFolderID: req.AudiobookRootFolderID, + Monitored: req.Monitored, + MonitorMode: req.MonitorMode, + MonitorLatestCount: req.MonitorLatestCount, + MonitorNewItems: req.MonitorNewItems, + SearchOnAdd: req.SearchOnAdd, + MediaType: req.MediaType, + }) if err != nil { - writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()}) + h.writeCreateAuthorError(w, r, err) return } - if author.ForeignID != "" { - if existing, _ := h.authors.GetByAnyForeignIDForUser(r.Context(), author.ForeignID, userID); existing != nil { - h.writeCanonicalAuthorConflict(w, existing, "author already exists") - return - } - } - if canonical, ambiguous, err := h.findCanonicalAuthorMatch(r.Context(), req.Name, author.Name); err != nil { - writeServerError(w, r, err) - return - } else if ambiguous { - writeJSON(w, http.StatusConflict, map[string]string{"error": "author name resolves ambiguously — merge manually"}) - return - } else if canonical != nil { - if canRelinkAuthorToUpstream(canonical) { - if err := h.relinkExistingAuthorToUpstream(r.Context(), canonical, author, req.Name, req.Monitored, monitorMode, monitorLatestCount, monitorNewItems, req.QualityProfileID, req.MetadataProfileID, req.RootFolderID, req.AudiobookRootFolderID); err != nil { - if isAuthorIdentityConflict(err) { - writeJSON(w, http.StatusConflict, map[string]string{"error": "upstream author already exists locally"}) - return - } - writeServerError(w, r, err) - return - } - mediaType := req.MediaType - if mediaType == "" { - mediaType = h.resolveDefaultMediaType(r.Context()) - } - // Finish mutating `canonical` (description clean-up) BEFORE spawning - // the async catalogue+profile refresh. fetchAuthorBooksAsync snapshots - // the author at spawn time and the goroutine now reads/writes profile - // fields (Description, ImageURL, ...); cleaning afterwards would race - // the snapshot read against this write (see fetchAuthorBooksAsync). - cleanAuthorDescription(canonical) - h.stampProviderMismatch(canonical) - h.fetchAuthorBooksAsync(canonical, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) - writeJSON(w, http.StatusOK, canonical) - return - } - h.writeCanonicalAuthorConflict(w, canonical, "author name already resolves to an existing author — confirm merge") + if !res.Created { + writeJSON(w, http.StatusOK, res.Author) return } - applyAuthorCreateOptions(author, req.Monitored, monitorMode, monitorLatestCount, req.QualityProfileID, req.MetadataProfileID, req.RootFolderID, req.AudiobookRootFolderID) - author.MonitorNewItems = monitorNewItems - - if err := h.authors.CreateForUser(r.Context(), author, auth.UserIDFromContext(r.Context())); err != nil { - slog.Error("create author failed", "foreign_id", req.ForeignID, "error", err) - if strings.Contains(err.Error(), "UNIQUE constraint failed") || errors.Is(err, db.ErrAuthorIdentifierConflict) { - if existing, _ := h.authors.GetByAnyForeignIDForUser(r.Context(), req.ForeignID, userID); existing != nil { - h.writeCanonicalAuthorConflict(w, existing, "author already exists") - return - } - if existing, _ := h.authors.GetByAnyForeignIDForUser(r.Context(), author.ForeignID, userID); existing != nil { - h.writeCanonicalAuthorConflict(w, existing, "author already exists") - return - } - writeJSON(w, http.StatusConflict, map[string]string{"error": "author already exists"}) - return - } - writeServerError(w, r, err) - return - } - h.recordAuthorCreateAlias(r.Context(), author, req.Name) - - // Persist any OL alternate names as alias rows so non-latin primary names - // (e.g. "村上春樹") get their latin-script alternates ("Haruki Murakami") - // indexed for release-name matching. - h.saveAlternateNames(r.Context(), author) - - // Resolve effective media type for books created under this author: - // explicit request value wins, else the global default.media_type - // setting, else ebook (backwards compat). - mediaType := req.MediaType - if mediaType == "" { - mediaType = h.resolveDefaultMediaType(r.Context()) - } - - // Clean the description BEFORE spawning the async refresh: the goroutine - // snapshots `author` and now reads/writes its profile fields (Description, - // ImageURL, ...). Cleaning after the spawn would race the snapshot read - // against this write (see fetchAuthorBooksAsync). - cleanAuthorDescription(author) - h.stampProviderMismatch(author) - - // Fetch and store books for this author. Always populate the catalogue; - // pass searchOnAdd so FetchAuthorBooks knows whether to also queue grabs. - h.fetchAuthorBooksAsync(author, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) - - telemetry.MarkFirst(r.Context(), h.settings, telemetry.SettingFirstAuthorAt) - writeJSON(w, http.StatusCreated, author) + writeJSON(w, http.StatusCreated, res.Author) } func cleanAuthorDescription(author *models.Author) { @@ -3085,402 +2985,18 @@ func (h *AuthorHandler) AddBook(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) return } - if req.ForeignBookID == "" { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "foreignBookId required"}) - return - } - switch req.MediaType { - case "", models.MediaTypeEbook, models.MediaTypeAudiobook, models.MediaTypeBoth: - default: - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "mediaType must be 'ebook', 'audiobook', or 'both'"}) - return - } - - ctx := r.Context() - - // bookCreated flips true once the poll loop confirms the requested - // book is in the DB. The orphan-cleanup defer below reads it on - // AddBook return — when false (poll timeout, ctx cancel, etc.) the - // just-created author row is deleted iff it has zero books. Fixes - // issue #667 bug 3. - bookCreated := false - - // authorWasJustCreated tracks whether this request inserted the author - // row (vs. found it already present). Used by both the orphan-cleanup - // defer and the direct-insert block below — when the author was just - // created the async catalogue sync may take longer than the 15s poll - // budget for prolific authors, so we synchronously persist the requested - // book to guarantee it exists before the cleanup defer runs (#804). - authorWasJustCreated := false - - // 0. Refuse to re-add a book the user already owns (#1227). Before this - // check the handler reused the existing row and force-monitored it at - // step 3, so re-adding an owned book silently flipped it back to wanted. - // Runs before any author creation or upstream fetch so a conflict has no - // side effects. Scoped like the library list the user is looking at - // (ListScopeUserID plus NULL owners): a NULL owned row is in that list, - // and an admin under tenancy sees every row, so both are conflicts. A - // non admin's request never sees another user's copy here; the guard - // after the poll covers that case. - userID := auth.UserIDFromContext(ctx) - scopeID := auth.ListScopeUserID(ctx) - if existing, err := h.books.GetByForeignIDVisibleTo(ctx, req.ForeignBookID, scopeID); err != nil { - writeServerError(w, r, err) - return - } else if existing != nil { - h.writeExistingBookConflict(w, existing) - return - } - - if req.ForeignAuthorID == "" { - resolved, outcome, err := h.resolveAuthorForBook(ctx, req.ForeignBookID) - if err != nil { - writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()}) - return - } - // The ISBN walk steps past a primary that timed out, so a fallback's - // record can win here only because the primary never answered. Adding - // it makes that provider the author's permanent link, and the same - // person is duplicated once the primary is back (#2117, #2612). Nothing - // has been written yet, so refuse and let a retry do the resolving. - if resolved != nil && !outcome.SafeToBind(resolved.Author.ForeignID) { - slog.Warn("AddBook: refusing to bind author to a fallback provider", - "foreignBookId", req.ForeignBookID, "primary", outcome.Primary, - "failed", outcome.FailureSummary(), "wouldHaveLinked", resolved.Author.ForeignID) - writePrimaryProviderUnavailable(w, outcome.Primary) - return - } - if resolved != nil { - // Rewrite the request so the existing fetch+poll flow targets the - // canonical provider's IDs. The user sees the canonical record (e.g. - // the OpenLibrary version) in their library; the original DNB record - // is dropped because bindery's author/book identity is single-source. - req.ForeignBookID = resolved.ForeignID - req.ForeignAuthorID = resolved.Author.ForeignID - if req.AuthorName == "" { - req.AuthorName = resolved.Author.Name - } - // The canonical id may itself already be in the library even though - // the id the client sent was not (#1227). - if existing, err := h.books.GetByForeignIDVisibleTo(ctx, req.ForeignBookID, scopeID); err != nil { - writeServerError(w, r, err) - return - } else if existing != nil { - h.writeExistingBookConflict(w, existing) - return - } - } else if req.AuthorName != "" { - // ISBN-based resolution failed (e.g. Google Books: author name, no - // author ID, no ISBN). Resolve the author by NAME — prefer one already - // in the library so we reuse the user's existing author instead of - // duplicating it; otherwise adopt OpenLibrary's canonical record. Keep - // the chosen edition (req.ForeignBookID) — the other providers don't - // have this book. - if existing := h.findLibraryAuthorByName(ctx, req.AuthorName); existing != nil { - req.ForeignAuthorID = existing.ForeignID - } else if canonical, cErr := h.meta.ResolveCanonicalAuthor(ctx, req.AuthorName); cErr == nil && canonical != nil && outcome.SafeToBind(canonical.ForeignID) { - // The canonical lookup is OpenLibrary's, which is a fallback - // when another provider is primary, so the same guard applies. - req.ForeignAuthorID = canonical.ForeignID - } - } - if req.ForeignAuthorID == "" && outcome.PrimaryFailed { - // No provider placed the author, but the primary never answered, - // so "add the author manually" is the wrong advice: the primary - // may well know this book. - writePrimaryProviderUnavailable(w, outcome.Primary) - return - } - if req.ForeignAuthorID == "" { - writeJSON(w, http.StatusUnprocessableEntity, map[string]string{ - "error": "Author metadata unavailable for this result. Add the author manually first (Authors → Add Author by name), then try again.", - }) - return - } - } - - // 1. Find or create the author (unmonitored if new so we don't auto-want all books). - author, _ := h.authors.GetByForeignIDForUser(ctx, req.ForeignAuthorID, userID) - if author == nil { - author, _ = h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignAuthorID, userID) - } - if author == nil { - name := req.AuthorName - if name == "" { - name = req.ForeignAuthorID - } - fetched, err := h.meta.GetAuthor(ctx, req.ForeignAuthorID) - if err != nil || fetched == nil { - fetched = &models.Author{ - ForeignID: req.ForeignAuthorID, - Name: name, - SortName: sortName(name), - MetadataProvider: "openlibrary", - } - } - fetched.Monitored = false - def := models.DefaultMetadataProfileID - fetched.MetadataProfileID = &def - - // Dedupe path: if a canonical provider (OL / Hardcover / …) is being - // added for a SortName previously persisted as a synthetic DNB-only - // row, migrate that row in place rather than creating a duplicate. - // The synthetic row was created because the DNB record had only an - // author name (no GND link, no OL coverage). Now that a canonical - // identity exists, collapse the two onto a single primary key so - // the user keeps one author with all their books attached. - if !strings.HasPrefix(fetched.ForeignID, "dnb:") { - if existing, lookupErr := h.authors.GetByDNBSyntheticName(ctx, fetched.SortName, userID); lookupErr == nil && existing != nil { - if err := h.authors.UpgradeSyntheticDNB(ctx, existing.ForeignID, fetched); err != nil { - slog.Debug("AddBook: upgrade synthetic DNB author failed", "from", existing.ForeignID, "to", fetched.ForeignID, "error", err) - } else { - // Re-fetch the row by its new canonical ForeignID so subsequent - // steps see the upgraded record (ID preserved). - if upgraded, getErr := h.authors.GetByForeignIDForUser(ctx, fetched.ForeignID, userID); getErr == nil && upgraded != nil { - author = upgraded - } - } - } - } - - // CreateForUser may collide with a concurrent request inserting the - // same author; the UNIQUE-constraint branch below recovers by - // re-fetching the row. authorWasJustCreated stays false on the race - // path so the orphan-cleanup defer never rolls back somebody else's - // author row (issue #667). - if author == nil { - // Add-book creates the author as a side effect, so it never carries - // an explicit monitor choice — take the install-wide default (#1666). - db.ApplyAuthorMonitorDefaults(ctx, h.settings, fetched) - if err := h.authors.CreateForUser(ctx, fetched, userID); err != nil { - if !strings.Contains(err.Error(), "UNIQUE constraint failed") && !errors.Is(err, db.ErrAuthorIdentifierConflict) { - writeServerError(w, r, err) - return - } - // Race: another request created it between our check and insert. - author, _ = h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignAuthorID, userID) - if author == nil { - writeJSON(w, http.StatusConflict, map[string]string{"error": "author already exists"}) - return - } - } else { - author = fetched - authorWasJustCreated = true - // No speculative catalogue fetch here (#1816). Adding a book - // creates its author as a side effect; the user picked ONE - // title, and pulling that author's whole bibliography in behind - // it is the "my collection went from 75 books to over 500" - // report — the thing nobody expects because adding a film to - // Radarr does not import the director's filmography. - // - // Nothing downstream needs it: the direct insert below creates - // the picked book synchronously, which is what makes the poll - // succeed and what keeps the orphan-cleanup defer from - // rolling the author back. The narrow case where that insert - // cannot produce the row has its own single-work fallback, - // just past the direct-insert block. - } - } - // Defer the orphan cleanup so cancellation paths inside the poll - // loop also benefit. Runs only after a CreateForUser this request. - if authorWasJustCreated { - defer h.cleanupOrphanIfNoBooks(author, &bookCreated) - } - } - if author == nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not resolve author"}) - return - } - - // 1b. Direct insert for the requested book. - // - // Originally added (#667) for DNB synthetic IDs, whose async sync returns - // zero books because DNB's public SRU has no author→works relationship. - // #804 widened this: for any author the request just created, the async - // catalogue sync can take longer than the 15 s poll budget (OpenLibrary - // took >32 s for a 175-work author in the bug report). When the poll - // times out, the orphan-cleanup defer deletes the author row — and the - // still-running goroutine then logs a FK-constraint failure for every - // book it tries to insert against the now-deleted author_id. - // - // Synchronously fetching and persisting the single requested record - // guarantees the poll succeeds on its first iteration AND that the - // cleanup defer sees a non-empty book list (so it keeps the author). - // The async sync still runs as a backfill for the rest of the catalogue; - // any UNIQUE collision against this row is silently tolerated. - // - // #1612 made the direct insert unconditional. When the author already - // EXISTED, AddBook used to skip it and rely entirely on a catalogue sync - // having created the row — but the sync can deterministically refuse a - // specific work (e.g. the work-level language sampled from the first few - // OpenLibrary editions falls outside the profile's allowed set, which is - // how heavily-translated works ended up permanently un-addable). Every - // attempt then polled 15 s for a row nothing would ever create and - // returned 404 "try again shortly" forever. An explicit add of one - // specific work is the strongest possible user signal and must not be - // vetoed by catalogue-sync heuristics; those heuristics still govern - // everything the user did NOT explicitly pick. - if existing, _ := h.books.GetByForeignID(ctx, req.ForeignBookID); existing == nil { - primary, err := h.meta.GetBook(ctx, req.ForeignBookID) - if err != nil { - slog.Warn("AddBook: direct fetch failed", - "foreignBookId", req.ForeignBookID, "error", err) - } else if primary != nil && h.directInsertTitleUsable(primary.Title, author.Name) { - primary.AuthorID = author.ID - // Tenancy (#1457): inherit the author's owner. Read off the author - // row rather than the request, because that row usually pre-exists - // this request (#1612) — the scoped lookup above is what makes it - // the correct owner either way. - primary.OwnerUserID = author.OwnerUserID - primary.Monitored = author.Monitored - if primary.Status == "" { - primary.Status = models.BookStatusWanted - } - // An explicit request choice wins over the provider's media type - // (#1397). Otherwise some providers (notably Google Books) don't - // set one; fall back to the global default so the row isn't - // created with an empty format (which would mis-route its - // indexer search). - if req.MediaType != "" { - primary.MediaType = req.MediaType - } else if primary.MediaType == "" { - primary.MediaType = h.resolveDefaultMediaType(ctx) - } - // Reuse a title-equivalent row under the same author instead of - // inserting a second one. The catalogue sync runs this same dedup - // (see the FindByAuthorAndDedupKey switch above), and skipping it - // here produced real duplicates: a Calibre-imported library holds - // the work under a `calibre:` foreign id, and OpenLibrary splits - // some works into separate ebook and audiobook Works that the sync - // merges into one media_type=both row — in both cases the - // requested foreign id has no row of its own, which is exactly the - // state that brings a user here. - match, ferr := h.books.FindByAuthorAndDedupKey(ctx, author.ID, primary.Title) - // A subtitle-collapsed dedup key (indexer.CanonicalDedupKey strips a - // ": subtitle" tail) merges every "Series: Volume" sibling onto one - // key. Adopting such a match would rebind the requested foreign id - // onto a *different* volume and — because adopt is a no-op when the - // row needs no field change — leave the poll below unable to find the - // requested id, returning 404 forever. When the requested work is a - // distinct volume of the matched row's series (same series, different - // sequence), skip the adopt and create a distinct row instead. - if ferr == nil && match != nil && !h.directInsertSeriesConflict(ctx, match.ID, primary.SeriesRefs) { - h.adoptDirectInsertMatch(ctx, match, primary, req.ForeignBookID) - } else if err := h.books.Create(ctx, primary); err != nil { - if !strings.Contains(err.Error(), "UNIQUE constraint failed") { - slog.Warn("AddBook: direct insert failed", - "foreignBookId", req.ForeignBookID, "error", err) - } - } else { - h.hydrateHardcoverEditions(ctx, primary, nil) - // Same post-create work every other creation path does - // (recommendations.go, series.go): check the library for a - // file we already have and link the book into its series. - // Without this the row is wanted-but-unchecked, so with - // searchOnAdd enabled Bindery re-downloads a book already on - // disk — the regression #940 and migration 026 exist to stop. - created := *primary - handleNewWantedBook(ctx, h.books, h.series, h.finder, created, author.Name) - } - } - } - - // 1c. Single-work fallback. The direct insert above covers the request in - // all but a couple of cases: the provider's book endpoint failed (#1612's - // OpenLibrary 502) or returned a record the title guard rejected, and the - // library has no row for the id either way. Ask the author endpoint for - // this ONE work instead — the same fetch the old speculative catalogue - // sync ran, restricted to the work the user actually picked, so the poll - // below can still succeed without the rest of the bibliography riding - // along (#1816). - if existing, _ := h.books.GetByForeignID(ctx, req.ForeignBookID); existing == nil { - // mediaType only fills a format the provider left blank, and step 3 - // below applies the request's explicit choice to whatever row the poll - // finds — so the default is the right value to pass here. It no longer - // decides whether the work is created at all: a single-work run is - // exempt from the strict media-type clamp (#1612). - h.fetchAuthorBooksAsync(author, catalogueSyncOptions{ - mediaType: h.resolveDefaultMediaType(ctx), - onlyForeignID: req.ForeignBookID, - }) - } - - // 2. Poll until the book appears (the single-work fallback, if it ran, - // creates it asynchronously). - deadline := time.Now().Add(15 * time.Second) - var book *models.Book - for { - b, _ := h.books.GetByForeignID(ctx, req.ForeignBookID) - if b != nil { - book = b - break - } - if time.Now().After(deadline) { - break - } - select { - case <-ctx.Done(): - writeJSON(w, http.StatusGatewayTimeout, map[string]string{"error": "request cancelled"}) - return - case <-time.After(500 * time.Millisecond): - } - } - - if book == nil { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "book not found after author sync — try again shortly"}) - return - } - // The poll looks the row up globally because books.foreign_id is UNIQUE - // across users, so under tenancy it can return another user's copy: the - // conflict gate above never saw it (scoped to this user), the direct - // insert skipped it, and until #1227 the update below then flipped that - // user's book to monitored. Refuse instead, with no row in the body since - // the caller is not allowed to see it. bookCreated stays false so the - // orphan cleanup defer removes any author row this request created. - if book.OwnerUserID != 0 && userID != 0 && book.OwnerUserID != userID { - writeJSON(w, http.StatusConflict, map[string]string{"error": "book is held by another user"}) - return - } - bookCreated = true - - // 3. Mark the book monitored (wanted). An explicit media-type choice is - // applied here too — the poll may have found a row created by the async - // catalogue sync (or one already in the library) carrying the default. - // Re-evaluate status on a change so e.g. adding an already-imported ebook - // as 'both' flips it back to wanted for the missing format (#1148). - book.Monitored = true - if req.MediaType != "" && book.MediaType != req.MediaType { - book.MediaType = req.MediaType - reevaluateBookStatus(book) - } - if err := h.books.Update(ctx, book); err != nil { - writeServerError(w, r, err) + res, err := h.addBookCore(r.Context(), addBookParams{ + ForeignBookID: req.ForeignBookID, + ForeignAuthorID: req.ForeignAuthorID, + AuthorName: req.AuthorName, + SearchOnAdd: req.SearchOnAdd, + MediaType: req.MediaType, + }) + if err != nil { + h.writeAddBookError(w, r, err) return } - - // 3b. Say so when this add went past the strict media-type policy (#1759). - // - // The policy is a catalogue-population rule, not a veto on what the user - // may own: the direct insert above never consults it, and the single-work - // fallback is exempt by #1612's rule that "an explicit add of one specific - // work must not be vetoed by catalogue-sync heuristics". Both of those are - // deliberate, because silently refusing an explicit user action is the - // worse of the two failures. - // - // What was missing is that it happened invisibly, so a user who turned the - // setting on to stop un-grabbable rows appearing had no way to learn that - // their own add was the exception. The setting's help text now says the - // same thing, which is the half most people will actually see. - h.logStrictMediaTypeBypass(ctx, book) - - // 4. Optionally trigger an indexer search. Use the process-lifecycle - // context so the search goroutine is cancelled on shutdown rather than - // running against context.Background(). See #846. - if req.SearchOnAdd && h.searcher != nil { - go h.searcher.SearchAndGrabBook(indexer.WithSearchOrigin(h.bgCtx(), indexer.OriginAuthor), *book) // #nosec G118 -- intentional: search must outlive the request - } - - writeJSON(w, http.StatusCreated, book) + writeJSON(w, http.StatusCreated, res.Book) } // logStrictMediaTypeBypass records an explicit add that the strict media-type diff --git a/internal/api/create_author_core.go b/internal/api/create_author_core.go new file mode 100644 index 000000000..285a35664 --- /dev/null +++ b/internal/api/create_author_core.go @@ -0,0 +1,255 @@ +package api + +import ( + "context" + "errors" + "log/slog" + "net/http" + "strings" + + "github.com/vavallee/bindery/internal/auth" + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/models" + "github.com/vavallee/bindery/internal/telemetry" +) + +// createAuthorParams is everything an Add Author needs, independent of HTTP. +// The Create handler fills it from the request body; request approval fills +// it itself. Ownership is not a field: the core reads the acting user from +// ctx, exactly as the handler always has. +type createAuthorParams struct { + ForeignID string + Name string + QualityProfileID *int64 + MetadataProfileID *int64 + RootFolderID *int64 + AudiobookRootFolderID *int64 + Monitored bool + MonitorMode *string + MonitorLatestCount *int + // Settable at add time as of the Add Author redesign: before this it + // could only be changed on the author afterwards, so "catalogue this + // author once and never let a refresh grow it" was not expressible + // when adding. Same validation as Update. + MonitorNewItems *string + SearchOnAdd bool + MediaType string + // SkipCatalogueSync creates or relinks the author without starting its + // catalogue sync. False keeps the handler's behaviour. + // + // The sync is everything that happens to the author after the row is + // written, so skipping it skips all of: the author profile refresh + // (description, image, disambiguation, ratings), the Calibre relink, the + // secondary provider identities, the catalogue works fetch and its + // Audible supplement, every book row the sync would create along with + // the library file check for each, the sync summary shown on the author + // page, and the search on add, which only ever runs inside that sync. + // Because SearchOnAdd would silently do nothing, the core refuses the + // combination with errCreateAuthorSearchNeedsSync. + SkipCatalogueSync bool +} + +// createAuthorResult is what an Add Author produced. +type createAuthorResult struct { + // Author is the created author, or the existing author relinked to the + // requested upstream record. + Author *models.Author + // Created is true when a new author row was inserted (the handler's 201) + // and false when an existing author was relinked instead (its 200). + Created bool +} + +// Refusals from createAuthorCore that carry no data. The Create handler maps +// each to a 400 with the body it has always answered with. +var ( + errCreateAuthorFieldsRequired = errors.New("create author: foreignAuthorId and authorName required") + errCreateAuthorInvalidMonitorNewItems = errors.New("create author: invalid monitorNewItems") + // errCreateAuthorSearchNeedsSync refuses SearchOnAdd with + // SkipCatalogueSync. The search runs inside the catalogue sync, so the + // pair would accept a search request and never search. Only a non HTTP + // caller can set SkipCatalogueSync, so the handler never returns it. + errCreateAuthorSearchNeedsSync = errors.New("create author: searchOnAdd requires the catalogue sync") +) + +var createAuthorErrorResponses = []struct { + err error + body string +}{ + {errCreateAuthorFieldsRequired, "foreignAuthorId and authorName required"}, + {errCreateAuthorInvalidMonitorNewItems, "invalid monitorNewItems"}, + {errCreateAuthorSearchNeedsSync, "searchOnAdd requires the catalogue sync"}, +} + +// createAuthorOptionError is an invalid monitorMode or monitorLatestCount. +// The handler answers 400 with the wrapped error's text. +type createAuthorOptionError struct { + Err error +} + +func (e *createAuthorOptionError) Error() string { return e.Err.Error() } +func (e *createAuthorOptionError) Unwrap() error { return e.Err } + +// createAuthorLookupError is a failed upstream author fetch. The handler +// answers 502 with the wrapped error's text. +type createAuthorLookupError struct { + Err error +} + +func (e *createAuthorLookupError) Error() string { return e.Err.Error() } +func (e *createAuthorLookupError) Unwrap() error { return e.Err } + +// authorConflictError is a 409: the author already exists, or its name +// resolves to one that does. Canonical, when set, is the existing author the +// handler returns so the client can offer to open it. +type authorConflictError struct { + Canonical *models.Author + Message string +} + +func (e *authorConflictError) Error() string { return "create author: " + e.Message } + +// writeCreateAuthorError answers a createAuthorCore error with the status +// and body the Create handler returned before the core was split out. +func (h *AuthorHandler) writeCreateAuthorError(w http.ResponseWriter, r *http.Request, err error) { + var conflict *authorConflictError + if errors.As(err, &conflict) { + h.writeCanonicalAuthorConflict(w, conflict.Canonical, conflict.Message) + return + } + var option *createAuthorOptionError + if errors.As(err, &option) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": option.Err.Error()}) + return + } + var lookup *createAuthorLookupError + if errors.As(err, &lookup) { + writeJSON(w, http.StatusBadGateway, map[string]string{"error": lookup.Err.Error()}) + return + } + for _, resp := range createAuthorErrorResponses { + if errors.Is(err, resp.err) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": resp.body}) + return + } + } + writeServerError(w, r, err) +} + +// createAuthorCore adds one author and starts its catalogue sync, or relinks +// an existing local author to the requested upstream record. It is Create +// without the HTTP layer. +func (h *AuthorHandler) createAuthorCore(ctx context.Context, req createAuthorParams) (createAuthorResult, error) { + if req.ForeignID == "" || req.Name == "" { + return createAuthorResult{}, errCreateAuthorFieldsRequired + } + if req.SearchOnAdd && req.SkipCatalogueSync { + return createAuthorResult{}, errCreateAuthorSearchNeedsSync + } + monitorMode, monitorLatestCount, err := h.resolveCreateMonitorOptions(ctx, req.MonitorMode, req.MonitorLatestCount) + monitorNewItems := models.DefaultAuthorMonitorNewItems + if req.MonitorNewItems != nil { + v := strings.TrimSpace(*req.MonitorNewItems) + if !models.IsAuthorMonitorNewItemsValid(v) { + return createAuthorResult{}, errCreateAuthorInvalidMonitorNewItems + } + monitorNewItems = v + } + if err != nil { + return createAuthorResult{}, &createAuthorOptionError{Err: err} + } + + // Check if already exists — use user-scoped lookup so this agrees with the + // author list, which filters by owner_user_id. A global GetByForeignID + // would block re-creation of authors orphaned under a different user ID. + userID := auth.UserIDFromContext(ctx) + existing, _ := h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignID, userID) + if existing != nil { + return createAuthorResult{}, &authorConflictError{Canonical: existing, Message: "author already exists"} + } + + author, err := h.fetchAuthorForCreate(ctx, req.ForeignID, req.Name) + if err != nil { + return createAuthorResult{}, &createAuthorLookupError{Err: err} + } + if author.ForeignID != "" { + if existing, _ := h.authors.GetByAnyForeignIDForUser(ctx, author.ForeignID, userID); existing != nil { + return createAuthorResult{}, &authorConflictError{Canonical: existing, Message: "author already exists"} + } + } + if canonical, ambiguous, err := h.findCanonicalAuthorMatch(ctx, req.Name, author.Name); err != nil { + return createAuthorResult{}, err + } else if ambiguous { + return createAuthorResult{}, &authorConflictError{Message: "author name resolves ambiguously — merge manually"} + } else if canonical != nil { + if canRelinkAuthorToUpstream(canonical) { + if err := h.relinkExistingAuthorToUpstream(ctx, canonical, author, req.Name, req.Monitored, monitorMode, monitorLatestCount, monitorNewItems, req.QualityProfileID, req.MetadataProfileID, req.RootFolderID, req.AudiobookRootFolderID); err != nil { + if isAuthorIdentityConflict(err) { + return createAuthorResult{}, &authorConflictError{Message: "upstream author already exists locally"} + } + return createAuthorResult{}, err + } + mediaType := req.MediaType + if mediaType == "" { + mediaType = h.resolveDefaultMediaType(ctx) + } + // Finish mutating `canonical` (description clean-up) BEFORE spawning + // the async catalogue+profile refresh. fetchAuthorBooksAsync snapshots + // the author at spawn time and the goroutine now reads/writes profile + // fields (Description, ImageURL, ...); cleaning afterwards would race + // the snapshot read against this write (see fetchAuthorBooksAsync). + cleanAuthorDescription(canonical) + h.stampProviderMismatch(canonical) + if !req.SkipCatalogueSync { + h.fetchAuthorBooksAsync(canonical, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) + } + return createAuthorResult{Author: canonical}, nil + } + return createAuthorResult{}, &authorConflictError{Canonical: canonical, Message: "author name already resolves to an existing author — confirm merge"} + } + applyAuthorCreateOptions(author, req.Monitored, monitorMode, monitorLatestCount, req.QualityProfileID, req.MetadataProfileID, req.RootFolderID, req.AudiobookRootFolderID) + author.MonitorNewItems = monitorNewItems + + if err := h.authors.CreateForUser(ctx, author, auth.UserIDFromContext(ctx)); err != nil { + slog.Error("create author failed", "foreign_id", req.ForeignID, "error", err) + if strings.Contains(err.Error(), "UNIQUE constraint failed") || errors.Is(err, db.ErrAuthorIdentifierConflict) { + if existing, _ := h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignID, userID); existing != nil { + return createAuthorResult{}, &authorConflictError{Canonical: existing, Message: "author already exists"} + } + if existing, _ := h.authors.GetByAnyForeignIDForUser(ctx, author.ForeignID, userID); existing != nil { + return createAuthorResult{}, &authorConflictError{Canonical: existing, Message: "author already exists"} + } + return createAuthorResult{}, &authorConflictError{Message: "author already exists"} + } + return createAuthorResult{}, err + } + h.recordAuthorCreateAlias(ctx, author, req.Name) + + // Persist any OL alternate names as alias rows so non-latin primary names + // (e.g. "村上春樹") get their latin-script alternates ("Haruki Murakami") + // indexed for release-name matching. + h.saveAlternateNames(ctx, author) + + // Resolve effective media type for books created under this author: + // explicit request value wins, else the global default.media_type + // setting, else ebook (backwards compat). + mediaType := req.MediaType + if mediaType == "" { + mediaType = h.resolveDefaultMediaType(ctx) + } + + // Clean the description BEFORE spawning the async refresh: the goroutine + // snapshots `author` and now reads/writes its profile fields (Description, + // ImageURL, ...). Cleaning after the spawn would race the snapshot read + // against this write (see fetchAuthorBooksAsync). + cleanAuthorDescription(author) + h.stampProviderMismatch(author) + + // Fetch and store books for this author. Always populate the catalogue; + // pass searchOnAdd so FetchAuthorBooks knows whether to also queue grabs. + if !req.SkipCatalogueSync { + h.fetchAuthorBooksAsync(author, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) + } + + telemetry.MarkFirst(ctx, h.settings, telemetry.SettingFirstAuthorAt) + return createAuthorResult{Author: author, Created: true}, nil +} diff --git a/internal/api/scan_walk.go b/internal/api/scan_walk.go index 5537df7e4..01844c44a 100644 --- a/internal/api/scan_walk.go +++ b/internal/api/scan_walk.go @@ -3,9 +3,7 @@ package api import ( "os" "path/filepath" - "regexp" "sort" - "strings" "github.com/vavallee/bindery/internal/importer" ) @@ -19,13 +17,6 @@ type scanUnit struct { isDir bool } -// discFolderRe matches the disc/part subfolder names ("CD1", "Disc 2", -// "Part 03", "Vol. 1", or a bare "1"/"02") that split ONE audiobook across -// several directories. When every subdirectory of a folder looks like a disc, -// the folder is a single multi-disc audiobook rather than a shelf of separate -// books. -var discFolderRe = regexp.MustCompile(`(?i)^(cd|dis[ck]|part|pt|vol|volume|book|chapter|ch)\s*[._-]?\s*\d+$|^\d{1,2}$`) - // walkUnitLimits bounds the recursive enumeration so pointing the scan at a // pathological tree can't stall the request or allocate without bound. const ( @@ -101,12 +92,12 @@ func enumerateImportUnits(root string, limit int) (units []scanUnit, truncated b return } // Every subdir a disc folder → one multi-disc audiobook. - if len(subdirs) > 0 && allDiscFolders(subdirs) { + if len(subdirs) > 0 && importer.AllDiscFolders(subdirs) { emit(dir, true) return } // A leaf folder of same-named ebooks → one book in several formats. - if len(subdirs) == 0 && len(ebookFiles) >= 2 && sameStem(ebookFiles) { + if len(subdirs) == 0 && len(ebookFiles) >= 2 && importer.SameStem(ebookFiles) { emit(dir, true) return } @@ -139,62 +130,3 @@ func enumerateImportUnits(root string, limit int) (units []scanUnit, truncated b walk(root, 0, true) return units, truncated } - -// allDiscFolders reports whether every directory in dirs is a disc/part folder -// (by name) that actually holds audio somewhere beneath it. Both conditions are -// required so a shelf of audiobook folders with numeric-ish names isn't collapsed -// into one book, and an empty "CD1" placeholder doesn't fake a multi-disc set. -func allDiscFolders(dirs []string) bool { - for _, d := range dirs { - if !discFolderRe.MatchString(filepath.Base(d)) { - return false - } - if !dirSubtreeHasAudio(d) { - return false - } - } - return len(dirs) > 0 -} - -// dirSubtreeHasAudio reports whether dir contains at least one audio file within -// a bounded number of entries, so a deep tree can't stall the disc-folder check. -func dirSubtreeHasAudio(dir string) bool { - const limit = 2000 - count := 0 - found := false - _ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { - if err != nil { - return nil - } - count++ - if count > limit { - return filepath.SkipAll - } - if !d.IsDir() && importer.IsAudioFile(p) { - found = true - return filepath.SkipAll - } - return nil - }) - return found -} - -// sameStem reports whether every file shares one normalised base name (the file -// name without its extension, lower-cased). Title.epub and Title.mobi share the -// stem "title"; Dune.epub and Foundation.epub do not. -func sameStem(files []string) bool { - stem := func(p string) string { - b := filepath.Base(p) - return strings.ToLower(strings.TrimSuffix(b, filepath.Ext(b))) - } - if len(files) == 0 { - return false - } - first := stem(files[0]) - for _, f := range files[1:] { - if stem(f) != first { - return false - } - } - return true -} diff --git a/internal/db/book_files_adoption.go b/internal/db/book_files_adoption.go new file mode 100644 index 000000000..d0b73a840 --- /dev/null +++ b/internal/db/book_files_adoption.go @@ -0,0 +1,25 @@ +package db + +import ( + "context" + "fmt" +) + +// UntrackFilePathForBook removes the book_files row for path only while it +// still belongs to bookID, then refreshes that book's status. It reports +// whether a row was removed. The file on disk is never touched. +// +// Library adoption's Undo uses it instead of UntrackFilePath: between the adopt +// and the undo the path may have been deleted with its book and registered to +// another one, and that book's row must survive the undo. +func (r *BookRepo) UntrackFilePathForBook(ctx context.Context, path string, bookID int64) (bool, error) { + out, err := r.exec.ExecContext(ctx, `DELETE FROM book_files WHERE path = ? AND book_id = ?`, path, bookID) + if err != nil { + return false, fmt.Errorf("untrack file path for book: %w", err) + } + n, err := out.RowsAffected() + if err != nil || n == 0 { + return false, err + } + return true, r.refreshBookStatus(ctx, bookID) +} diff --git a/internal/db/migrations/088_unmatched_units.sql b/internal/db/migrations/088_unmatched_units.sql new file mode 100644 index 000000000..19a9aee57 --- /dev/null +++ b/internal/db/migrations/088_unmatched_units.sql @@ -0,0 +1,88 @@ +-- +migrate Up + +-- Library adoption: the files a library scan could not match, one row per +-- book rather than per file. +-- +-- Until now the scan wrote its unmatched files into the library.lastScan +-- settings blob, capped at 1000 entries, one entry per file. A row there had +-- no identity, so nothing could be decided about it: no ignore state, no +-- grouping of a 193 track audiobook into one book (#2547), and no way to mark +-- a file as dealt with. This table gives each unit a stable identity keyed by +-- its path, so a decision (adopted, ignored) survives the next scan. +-- +-- state: +-- pending the scan still finds this unit and nobody has decided yet +-- adopting an adopt request holds the row (compare and swap claim) +-- adopted files registered against book_id by an admin +-- undoing an undo request holds the row +-- ignored an admin said this unit is not a book to track +-- +-- created_book_id and created_author_id are set only when the adopt request +-- itself inserted that row, which is what lets Undo remove them again without +-- touching a book or author that existed before. registered_paths_json lists +-- exactly the book_files rows the adopt inserted, each as its path and the +-- book it was registered to, so Undo never removes a row that has since moved +-- to another book. created_book_fingerprint records the created book as the +-- adoption left it; if it differs at Undo, someone has started using the book +-- and Undo keeps it. +-- +-- An adopt writes each of these as soon as it exists, before the next side +-- effect, so a request that dies part way leaves a row that says exactly what +-- to reverse. claimed_at dates the claim for that recovery; updated_at cannot, +-- because every scan refreshes it. +-- +-- member_paths_json holds the files that make up the unit. Adopt re-checks +-- each one on disk before registering anything, because the row may be hours +-- old by then. +-- +-- Times are written by the repository in one fixed width UTC layout so the +-- purge and claim recovery comparisons can compare them as text. + +CREATE TABLE unmatched_units ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + unit_path TEXT NOT NULL UNIQUE, + unit_kind TEXT NOT NULL CHECK(unit_kind IN ('file', 'folder')), + format TEXT NOT NULL CHECK(format IN ('ebook', 'audiobook')), + file_count INTEGER NOT NULL DEFAULT 1, + size_bytes INTEGER NOT NULL DEFAULT 0, + root_path TEXT NOT NULL DEFAULT '', + rel_path TEXT NOT NULL DEFAULT '', + author_folder TEXT NOT NULL DEFAULT '', + parsed_title TEXT NOT NULL DEFAULT '', + parsed_author TEXT NOT NULL DEFAULT '', + reason TEXT NOT NULL DEFAULT '', + search_key TEXT NOT NULL DEFAULT '', + member_paths_json TEXT NOT NULL DEFAULT '[]', + candidates_json TEXT NOT NULL DEFAULT '[]', + top_score REAL NOT NULL DEFAULT 0, + state TEXT NOT NULL DEFAULT 'pending' + CHECK(state IN ('pending', 'adopting', 'adopted', 'undoing', 'ignored')), + book_id INTEGER REFERENCES books(id) ON DELETE SET NULL, + created_book_id INTEGER REFERENCES books(id) ON DELETE SET NULL, + created_author_id INTEGER REFERENCES authors(id) ON DELETE SET NULL, + registered_paths_json TEXT NOT NULL DEFAULT '[]', + created_book_fingerprint TEXT NOT NULL DEFAULT '', + scan_generation INTEGER NOT NULL DEFAULT 0, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + resolved_at TEXT, + claimed_at TEXT, + updated_at TEXT NOT NULL +); + +-- The default list: pending units, best suggestion first. +CREATE INDEX idx_unmatched_units_state_score ON unmatched_units(state, top_score DESC, rel_path); +-- The folder rail and the folder filter. +CREATE INDEX idx_unmatched_units_state_folder ON unmatched_units(state, author_folder, rel_path); +-- The reason facet and filter. +CREATE INDEX idx_unmatched_units_state_reason ON unmatched_units(state, reason); +-- Undo's "does any other row still point at this book" check. +CREATE INDEX idx_unmatched_units_book ON unmatched_units(book_id); + +-- +migrate Down + +DROP INDEX IF EXISTS idx_unmatched_units_book; +DROP INDEX IF EXISTS idx_unmatched_units_state_reason; +DROP INDEX IF EXISTS idx_unmatched_units_state_folder; +DROP INDEX IF EXISTS idx_unmatched_units_state_score; +DROP TABLE IF EXISTS unmatched_units; diff --git a/internal/db/unmatched_units.go b/internal/db/unmatched_units.go new file mode 100644 index 000000000..be195d102 --- /dev/null +++ b/internal/db/unmatched_units.go @@ -0,0 +1,713 @@ +package db + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/vavallee/bindery/internal/textutil" +) + +// Unmatched unit states. See migration 088 for what each one means. +const ( + UnmatchedStatePending = "pending" + UnmatchedStateAdopting = "adopting" + UnmatchedStateAdopted = "adopted" + UnmatchedStateUndoing = "undoing" + UnmatchedStateIgnored = "ignored" +) + +// Unit kinds: a single file (an ebook, or loose audio at a library root), or +// a folder that stands for one book (an audiobook folder or a disc set). +const ( + UnmatchedKindFile = "file" + UnmatchedKindFolder = "folder" +) + +const ( + // reconcileChunkSize bounds how long one scan tail holds SQLite's single + // writer lock (P3). Imports and the queue get the lock between chunks. + reconcileChunkSize = 500 + // unmatchedPurgeAge is how long an ignored or adopted row may go unseen by + // a scan before it is purged. + unmatchedPurgeAge = 30 * 24 * time.Hour + // UnmatchedClaimTimeout is how old a claim must be before it counts as + // abandoned by a request that died. No live request holds a row anywhere + // near this long. + UnmatchedClaimTimeout = 15 * time.Minute +) + +// unitTimeLayout is fixed width, so two stored times compare correctly as +// text. RFC3339Nano trims trailing zeros, and "05Z" sorts after "05.1Z". +const unitTimeLayout = "2006-01-02T15:04:05.000000000Z" + +func unitTime(t time.Time) string { return t.UTC().Format(unitTimeLayout) } + +// UnmatchedCandidate is one catalogue book the scan thought this unit might +// be, with the title similarity that put it there. +type UnmatchedCandidate struct { + BookID int64 `json:"bookId"` + Score float64 `json:"score"` +} + +// UnmatchedUnitScan is what a library scan reports for one unit. +type UnmatchedUnitScan struct { + UnitPath string + UnitKind string + Format string + FileCount int + SizeBytes int64 + RootPath string + RelPath string + AuthorFolder string + ParsedTitle string + ParsedAuthor string + Reason string + MemberPaths []string + Candidates []UnmatchedCandidate +} + +// RegisteredFile is one book_files row an adoption inserted: its path and the +// book it was registered to. Undo removes the row only while it still belongs +// to that book. +type RegisteredFile struct { + Path string `json:"path"` + BookID int64 `json:"bookId"` +} + +// UnmatchedUnit is one stored row. +type UnmatchedUnit struct { + ID int64 + UnitPath string + UnitKind string + Format string + FileCount int + SizeBytes int64 + RootPath string + RelPath string + AuthorFolder string + ParsedTitle string + ParsedAuthor string + Reason string + MemberPaths []string + Candidates []UnmatchedCandidate + TopScore float64 + State string + BookID int64 + CreatedBookID int64 + CreatedAuthorID int64 + // CreatedBookFingerprint is BookFingerprint of the created book as the + // adoption left it; empty when no book was created or the adoption did + // not get that far. + CreatedBookFingerprint string + Registered []RegisteredFile + ScanGeneration int64 + FirstSeenAt time.Time + LastSeenAt time.Time + ResolvedAt *time.Time + ClaimedAt *time.Time + // ClaimToken identifies the request holding an adopting or undoing row. + // Every write that belongs to that request is scoped to it, so a request + // whose claim was recovered and taken by another cannot write into it. + ClaimToken string +} + +// ReconcileScanOptions tunes one ReconcileScan call. +type ReconcileScanOptions struct { + // StartedAt is when the scan read the tracked paths. An adopted row the + // scan still found unmatched returns to pending only when it was adopted + // before this, so an adopt that lands while a long walk is running is not + // undone by that walk's stale view. + StartedAt time.Time + // SkipDeletion keeps every existing row: no unseen pending row is removed + // and nothing is purged. The scanner sets it when the scan was truncated, + // because a unit beyond the cap was not unseen, only unlisted. (A scan that + // found no files at all, or failed, does not call ReconcileScan.) + SkipDeletion bool + // RootsWithFiles lists the scanned roots that produced at least one book + // file. Ignored rows are purged only under these, so an unmounted + // audiobook root does not forget its ignores while the library root scans. + RootsWithFiles []string + // ConfiguredRoots lists every root the scan was configured with, files or + // not. An old ignored row under a root that is no longer configured at + // all is purged too; no future scan would ever see its files again. + ConfiguredRoots []string + // Now is the clock; zero means time.Now. + Now time.Time +} + +// ReconcileScanResult reports what a reconcile did and the counts after it. +type ReconcileScanResult struct { + Generation int64 + Upserted int + RemovedPending int64 + Purged int64 + Pending int + Ignored int +} + +// UnmatchedUnitRepo stores library adoption rows. +type UnmatchedUnitRepo struct { + db *sql.DB +} + +// NewUnmatchedUnitRepo returns a repo over database. +func NewUnmatchedUnitRepo(database *sql.DB) *UnmatchedUnitRepo { + return &UnmatchedUnitRepo{db: database} +} + +const upsertUnmatchedUnitSQL = ` +INSERT INTO unmatched_units ( + unit_path, unit_kind, format, file_count, size_bytes, root_path, rel_path, + author_folder, parsed_title, parsed_author, reason, search_key, + member_paths_json, candidates_json, top_score, scan_generation, + first_seen_at, last_seen_at, updated_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(unit_path) DO UPDATE SET + unit_kind = excluded.unit_kind, + format = excluded.format, + file_count = excluded.file_count, + size_bytes = excluded.size_bytes, + root_path = excluded.root_path, + rel_path = excluded.rel_path, + author_folder = excluded.author_folder, + parsed_title = excluded.parsed_title, + parsed_author = excluded.parsed_author, + reason = excluded.reason, + search_key = excluded.search_key, + member_paths_json = excluded.member_paths_json, + candidates_json = excluded.candidates_json, + top_score = excluded.top_score, + scan_generation = excluded.scan_generation, + last_seen_at = excluded.last_seen_at, + updated_at = excluded.updated_at, + -- An adoption whose files the scan no longer sees as tracked has been + -- undone some other way (the book was deleted, a file row was removed), + -- so the unit is a decision to make again. Ignored stays ignored, and a + -- row adopted after this scan started is left alone. + state = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN 'pending' ELSE unmatched_units.state END, + book_id = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN NULL ELSE unmatched_units.book_id END, + created_book_id = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN NULL ELSE unmatched_units.created_book_id END, + created_author_id = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN NULL ELSE unmatched_units.created_author_id END, + registered_paths_json = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN '[]' ELSE unmatched_units.registered_paths_json END, + created_book_fingerprint = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN '' ELSE unmatched_units.created_book_fingerprint END, + resolved_at = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN NULL ELSE unmatched_units.resolved_at END` + +// ReconcileScan records one scan's units. Upserts are keyed by unit_path and +// committed every reconcileChunkSize rows (P3). A decision already taken on a +// row is kept: the upsert never moves an ignored row, and moves an adopted row +// back to pending only as described in upsertUnmatchedUnitSQL, and never +// touches a row an adopt or undo holds (those are recovered with their side +// effects by the adoption handler, not here). After the upserts, one short +// transaction removes pending rows this scan did not see, purges adopted rows +// whose book is gone, and purges ignored rows unseen for 30 days under a root +// that produced files. The generation number keeps this correct if +// the process dies between chunks: the next scan's generation supersedes it. +func (r *UnmatchedUnitRepo) ReconcileScan(ctx context.Context, units []UnmatchedUnitScan, opts ReconcileScanOptions) (ReconcileScanResult, error) { + now := opts.Now + if now.IsZero() { + now = time.Now() + } + started := opts.StartedAt + if started.IsZero() { + started = now + } + var res ReconcileScanResult + if err := r.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(scan_generation), 0) + 1 FROM unmatched_units`).Scan(&res.Generation); err != nil { + return res, fmt.Errorf("unmatched units: next generation: %w", err) + } + nowText := unitTime(now) + startedText := unitTime(started) + + for start := 0; start < len(units); start += reconcileChunkSize { + end := min(start+reconcileChunkSize, len(units)) + if err := r.upsertChunk(ctx, units[start:end], res.Generation, nowText, startedText); err != nil { + return res, err + } + res.Upserted += end - start + } + + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return res, fmt.Errorf("unmatched units: begin cleanup: %w", err) + } + defer func() { _ = tx.Rollback() }() + if !opts.SkipDeletion { + out, err := tx.ExecContext(ctx, + `DELETE FROM unmatched_units WHERE state = 'pending' AND scan_generation < ?`, res.Generation) + if err != nil { + return res, fmt.Errorf("unmatched units: remove unseen pending: %w", err) + } + res.RemovedPending, _ = out.RowsAffected() + // An adopted row's files are tracked, so no scan sees it again; its age + // says nothing. It stays while its book exists, which is what keeps + // Undo available, and goes once the book is deleted. + out, err = tx.ExecContext(ctx, `DELETE FROM unmatched_units WHERE state = 'adopted' AND book_id IS NULL`) + if err != nil { + return res, fmt.Errorf("unmatched units: purge orphaned adoptions: %w", err) + } + purged, _ := out.RowsAffected() + res.Purged += purged + if cond, args := ignoredPurgeScope(opts); cond != "" { + //nolint:gosec // G202: cond is generated ? placeholders; every root is bound + out, err = tx.ExecContext(ctx, + `DELETE FROM unmatched_units WHERE state = 'ignored' AND last_seen_at < ? AND (`+cond+`)`, + append([]any{unitTime(now.Add(-unmatchedPurgeAge))}, args...)...) + if err != nil { + return res, fmt.Errorf("unmatched units: purge old ignores: %w", err) + } + purged, _ = out.RowsAffected() + res.Purged += purged + } + } + if err := tx.Commit(); err != nil { + return res, fmt.Errorf("unmatched units: commit cleanup: %w", err) + } + + summary, err := r.Summary(ctx) + if err != nil { + return res, err + } + res.Pending = summary.Pending + res.Ignored = summary.Ignored + return res, nil +} + +func (r *UnmatchedUnitRepo) upsertChunk(ctx context.Context, chunk []UnmatchedUnitScan, generation int64, nowText, startedText string) error { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("unmatched units: begin chunk: %w", err) + } + defer func() { _ = tx.Rollback() }() + stmt, err := tx.PrepareContext(ctx, upsertUnmatchedUnitSQL) + if err != nil { + return fmt.Errorf("unmatched units: prepare upsert: %w", err) + } + defer func() { _ = stmt.Close() }() + for i := range chunk { + u := &chunk[i] + members, err := json.Marshal(nonNilStrings(u.MemberPaths)) + if err != nil { + return fmt.Errorf("unmatched units: encode members: %w", err) + } + cands := u.Candidates + if cands == nil { + cands = []UnmatchedCandidate{} + } + candJSON, err := json.Marshal(cands) + if err != nil { + return fmt.Errorf("unmatched units: encode candidates: %w", err) + } + var top float64 + for _, c := range cands { + top = max(top, c.Score) + } + fileCount := max(u.FileCount, 1) + searchKey := textutil.FoldForSearch(u.ParsedTitle + " " + u.ParsedAuthor + " " + u.RelPath) + if _, err := stmt.ExecContext(ctx, + u.UnitPath, u.UnitKind, u.Format, fileCount, u.SizeBytes, u.RootPath, u.RelPath, + u.AuthorFolder, u.ParsedTitle, u.ParsedAuthor, u.Reason, searchKey, + string(members), string(candJSON), top, generation, + nowText, nowText, nowText, + startedText, startedText, startedText, startedText, startedText, startedText, startedText, + ); err != nil { + return fmt.Errorf("unmatched units: upsert %q: %w", u.UnitPath, err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("unmatched units: commit chunk: %w", err) + } + return nil +} + +// ignoredPurgeScope builds the root condition for purging old ignored rows: +// under a root that produced files in this scan, or under a root that is no +// longer configured at all. A configured root that produced no files (an +// unmounted volume) is in neither and keeps its rows. +func ignoredPurgeScope(opts ReconcileScanOptions) (string, []any) { + var parts []string + var args []any + placeholders := func(roots []string) string { + for _, r := range roots { + args = append(args, r) + } + return strings.TrimSuffix(strings.Repeat("?,", len(roots)), ",") + } + if len(opts.RootsWithFiles) > 0 { + parts = append(parts, "root_path IN ("+placeholders(opts.RootsWithFiles)+")") + } + if len(opts.ConfiguredRoots) > 0 { + parts = append(parts, "root_path NOT IN ("+placeholders(opts.ConfiguredRoots)+")") + } + return strings.Join(parts, " OR "), args +} + +func nonNilStrings(s []string) []string { + if s == nil { + return []string{} + } + return s +} + +// UnmatchedSummary is the headline counts, cheap enough for a nav badge. +type UnmatchedSummary struct { + Pending int `json:"pending"` + PendingFiles int `json:"pendingFiles"` + Ignored int `json:"ignored"` + Adopted int `json:"adopted"` +} + +// Summary counts rows by state. The adopting and undoing states count with +// the state they came from, since that is what the row still is to a viewer. +func (r *UnmatchedUnitRepo) Summary(ctx context.Context) (UnmatchedSummary, error) { + var s UnmatchedSummary + rows, err := r.db.QueryContext(ctx, + `SELECT state, COUNT(*), COALESCE(SUM(file_count), 0) FROM unmatched_units GROUP BY state`) + if err != nil { + return s, fmt.Errorf("unmatched units: summary: %w", err) + } + defer rows.Close() + for rows.Next() { + var state string + var n, files int + if err := rows.Scan(&state, &n, &files); err != nil { + return s, fmt.Errorf("unmatched units: summary scan: %w", err) + } + switch state { + case UnmatchedStatePending, UnmatchedStateAdopting: + s.Pending += n + s.PendingFiles += files + case UnmatchedStateAdopted, UnmatchedStateUndoing: + s.Adopted += n + case UnmatchedStateIgnored: + s.Ignored += n + } + } + return s, rows.Err() +} + +const unmatchedUnitColumns = `id, unit_path, unit_kind, format, file_count, size_bytes, root_path, rel_path, + author_folder, parsed_title, parsed_author, reason, member_paths_json, candidates_json, top_score, + state, book_id, created_book_id, created_author_id, registered_paths_json, created_book_fingerprint, + scan_generation, first_seen_at, last_seen_at, resolved_at, claimed_at` + +func scanUnmatchedUnit(scan func(...any) error) (*UnmatchedUnit, error) { + var u UnmatchedUnit + var members, cands, registered string + var bookID, createdBook, createdAuthor sql.NullInt64 + var first, last, resolved, claimed sql.NullString + if err := scan(&u.ID, &u.UnitPath, &u.UnitKind, &u.Format, &u.FileCount, &u.SizeBytes, &u.RootPath, &u.RelPath, + &u.AuthorFolder, &u.ParsedTitle, &u.ParsedAuthor, &u.Reason, &members, &cands, &u.TopScore, + &u.State, &bookID, &createdBook, &createdAuthor, ®istered, &u.CreatedBookFingerprint, + &u.ScanGeneration, &first, &last, &resolved, &claimed); err != nil { + return nil, err + } + u.BookID, u.CreatedBookID, u.CreatedAuthorID = bookID.Int64, createdBook.Int64, createdAuthor.Int64 + if err := json.Unmarshal([]byte(members), &u.MemberPaths); err != nil { + return nil, fmt.Errorf("unmatched unit %d members: %w", u.ID, err) + } + if err := json.Unmarshal([]byte(cands), &u.Candidates); err != nil { + return nil, fmt.Errorf("unmatched unit %d candidates: %w", u.ID, err) + } + if err := json.Unmarshal([]byte(registered), &u.Registered); err != nil { + return nil, fmt.Errorf("unmatched unit %d registered paths: %w", u.ID, err) + } + u.FirstSeenAt = parseFlexibleTimeValue(first, "unmatched_units.first_seen_at") + u.LastSeenAt = parseFlexibleTimeValue(last, "unmatched_units.last_seen_at") + if t, err := parseFlexibleTime(resolved); err == nil { + u.ResolvedAt = t + } + if claimed.Valid { + u.ClaimToken = claimed.String + stamp, _, _ := strings.Cut(claimed.String, "#") + if t, err := parseFlexibleTime(sql.NullString{String: stamp, Valid: true}); err == nil { + u.ClaimedAt = t + } + } + return &u, nil +} + +// Get returns one row, or nil when there is none. +func (r *UnmatchedUnitRepo) Get(ctx context.Context, id int64) (*UnmatchedUnit, error) { + row := r.db.QueryRowContext(ctx, `SELECT `+unmatchedUnitColumns+` FROM unmatched_units WHERE id = ?`, id) + u, err := scanUnmatchedUnit(row.Scan) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("unmatched units: get %d: %w", id, err) + } + return u, nil +} + +// Claim moves a row into adopting or undoing only if it is still in from, and +// returns the claim token that scopes every later write of this request ("" if +// another request got there first). The token starts with the fixed width +// claim time, so it still orders as a time for StaleClaims. +func (r *UnmatchedUnitRepo) Claim(ctx context.Context, id int64, from, to string) (string, error) { + nonce := make([]byte, 8) + if _, err := rand.Read(nonce); err != nil { + return "", fmt.Errorf("unmatched units: claim token: %w", err) + } + now := unitTime(time.Now()) + token := now + "#" + hex.EncodeToString(nonce) + out, err := r.db.ExecContext(ctx, + `UPDATE unmatched_units SET state = ?, claimed_at = ?, updated_at = ? WHERE id = ? AND state = ?`, + to, token, now, id, from) + if err != nil { + return "", fmt.Errorf("unmatched units: claim %d %s to %s: %w", id, from, to, err) + } + if n, err := out.RowsAffected(); err != nil || n != 1 { + return "", err + } + return token, nil +} + +// ReleaseClaim moves a row this request holds (token) back to a settled state +// without touching its record. +func (r *UnmatchedUnitRepo) ReleaseClaim(ctx context.Context, id int64, from, to, token string) (bool, error) { + out, err := r.db.ExecContext(ctx, + `UPDATE unmatched_units SET state = ?, claimed_at = NULL, updated_at = ? WHERE id = ? AND state = ? AND claimed_at IS ?`, + to, unitTime(time.Now()), id, from, token) + if err != nil { + return false, fmt.Errorf("unmatched units: release %d: %w", id, err) + } + n, err := out.RowsAffected() + return n == 1, err +} + +// ClaimState moves a row from one state to another only if it is still in +// the first. It is the compare and swap every adopt, undo and ignore goes +// through (S9): a single UPDATE is atomic in SQLite, so of two requests racing +// for one row exactly one sees a row affected. +func (r *UnmatchedUnitRepo) ClaimState(ctx context.Context, id int64, from, to string) (bool, error) { + if to == UnmatchedStateAdopting || to == UnmatchedStateUndoing { + token, err := r.Claim(ctx, id, from, to) + return token != "", err + } + now := unitTime(time.Now()) + out, err := r.db.ExecContext(ctx, + `UPDATE unmatched_units SET state = ?, claimed_at = NULL, updated_at = ? WHERE id = ? AND state = ?`, + to, now, id, from) + if err != nil { + return false, fmt.Errorf("unmatched units: claim %d %s to %s: %w", id, from, to, err) + } + n, err := out.RowsAffected() + return n == 1, err +} + +// AdoptionRecord is what an adopt has written so far, and what undo or claim +// recovery reverses. +type AdoptionRecord struct { + BookID int64 + CreatedBookID int64 + CreatedAuthorID int64 + CreatedBookFingerprint string + Registered []RegisteredFile +} + +func (rec AdoptionRecord) registeredJSON() (string, error) { + files := rec.Registered + if files == nil { + files = []RegisteredFile{} + } + b, err := json.Marshal(files) + return string(b), err +} + +// RecordAdoptionProgress writes what an adopt has done so far into the row it +// holds, before its next side effect, so a request that dies part way leaves +// the row saying exactly what to reverse. It reports false when the row is no +// longer held by an adopt. +func (r *UnmatchedUnitRepo) RecordAdoptionProgress(ctx context.Context, id int64, token string, rec AdoptionRecord) (bool, error) { + return r.writeAdoption(ctx, id, token, rec, false) +} + +// CompleteAdoption moves a claimed row to adopted with the full record. It +// reports false when the row is no longer held by the adopt. +func (r *UnmatchedUnitRepo) CompleteAdoption(ctx context.Context, id int64, token string, rec AdoptionRecord) (bool, error) { + return r.writeAdoption(ctx, id, token, rec, true) +} + +func (r *UnmatchedUnitRepo) writeAdoption(ctx context.Context, id int64, token string, rec AdoptionRecord, complete bool) (bool, error) { + paths, err := rec.registeredJSON() + if err != nil { + return false, err + } + now := unitTime(time.Now()) + query := `UPDATE unmatched_units + SET book_id = ?, created_book_id = ?, created_author_id = ?, registered_paths_json = ?, + created_book_fingerprint = ?, updated_at = ? + WHERE id = ? AND state = 'adopting' AND claimed_at IS ?` + args := []any{nullID(rec.BookID), nullID(rec.CreatedBookID), nullID(rec.CreatedAuthorID), paths, rec.CreatedBookFingerprint, now, id, token} + if complete { + query = `UPDATE unmatched_units + SET state = 'adopted', book_id = ?, created_book_id = ?, created_author_id = ?, registered_paths_json = ?, + created_book_fingerprint = ?, updated_at = ?, resolved_at = ?, claimed_at = NULL + WHERE id = ? AND state = 'adopting' AND claimed_at IS ?` + args = []any{nullID(rec.BookID), nullID(rec.CreatedBookID), nullID(rec.CreatedAuthorID), paths, rec.CreatedBookFingerprint, now, now, id, token} + } + out, err := r.db.ExecContext(ctx, query, args...) + if err != nil { + return false, fmt.Errorf("unmatched units: write adoption %d: %w", id, err) + } + n, err := out.RowsAffected() + return n == 1, err +} + +// ResetToPending returns a row the request holding token has in state from +// (undoing, or adopting) to pending and clears the adoption. Call it only once +// every side effect has been reversed; the record is what a retry needs. +func (r *UnmatchedUnitRepo) ResetToPending(ctx context.Context, id int64, from, token string) (bool, error) { + out, err := r.db.ExecContext(ctx, ` + UPDATE unmatched_units + SET state = 'pending', book_id = NULL, created_book_id = NULL, created_author_id = NULL, + registered_paths_json = '[]', created_book_fingerprint = '', resolved_at = NULL, claimed_at = NULL, + updated_at = ? + WHERE id = ? AND state = ? AND claimed_at IS ?`, unitTime(time.Now()), id, from, token) + if err != nil { + return false, fmt.Errorf("unmatched units: reset %d: %w", id, err) + } + n, err := out.RowsAffected() + return n == 1, err +} + +// CompleteUndo returns an undoing row to pending. +func (r *UnmatchedUnitRepo) CompleteUndo(ctx context.Context, id int64, token string) (bool, error) { + return r.ResetToPending(ctx, id, UnmatchedStateUndoing, token) +} + +// StaleClaims lists rows an adopt or undo claimed before the cutoff: requests +// that died holding them. +func (r *UnmatchedUnitRepo) StaleClaims(ctx context.Context, before time.Time) ([]UnmatchedUnit, error) { + rows, err := r.db.QueryContext(ctx, + `SELECT `+unmatchedUnitColumns+` FROM unmatched_units + WHERE state IN ('adopting', 'undoing') AND (claimed_at IS NULL OR claimed_at < ?)`, unitTime(before)) + if err != nil { + return nil, fmt.Errorf("unmatched units: stale claims: %w", err) + } + defer rows.Close() + var out []UnmatchedUnit + for rows.Next() { + u, err := scanUnmatchedUnit(rows.Scan) + if err != nil { + return nil, fmt.Errorf("unmatched units: stale claims scan: %w", err) + } + out = append(out, *u) + } + return out, rows.Err() +} + +// BookFingerprint summarises the parts of a book a person changes by using it: +// its row (updated_at moves on any edit, monitor toggle or refresh), its +// Calibre link, and the history, downloads, pending releases, blocklist +// entries, series links, editions and provider identifiers hanging off it. +// An empty string means the book does not exist. +// +// It errs towards "used". A status refresh, a cover write or a manual author +// refresh also move updated_at without anyone touching the book, and Undo then +// keeps a book it could have removed. That is the safe direction: an extra +// unmonitored book is visible and deletable, a removed one that was in use is +// not recoverable. Do not narrow it to chase those. +func (r *UnmatchedUnitRepo) BookFingerprint(ctx context.Context, bookID int64) (string, error) { + var monitored bool + var updated sql.NullString + var calibreID sql.NullInt64 + var history, downloads, pending, blocklist, series, editions, identifiers int + err := r.db.QueryRowContext(ctx, ` + SELECT b.monitored, b.updated_at, b.calibre_id, + (SELECT COUNT(*) FROM history WHERE book_id = b.id), + (SELECT COUNT(*) FROM downloads WHERE book_id = b.id), + (SELECT COUNT(*) FROM pending_releases WHERE book_id = b.id), + (SELECT COUNT(*) FROM blocklist WHERE book_id = b.id), + (SELECT COUNT(*) FROM series_books WHERE book_id = b.id), + (SELECT COUNT(*) FROM editions WHERE book_id = b.id), + (SELECT COUNT(*) FROM book_identifiers WHERE book_id = b.id) + FROM books b WHERE b.id = ?`, bookID).Scan(&monitored, &updated, &calibreID, + &history, &downloads, &pending, &blocklist, &series, &editions, &identifiers) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("unmatched units: book fingerprint %d: %w", bookID, err) + } + return fmt.Sprintf("m=%t u=%s c=%d h=%d d=%d p=%d b=%d s=%d e=%d i=%d", + monitored, updated.String, calibreID.Int64, history, downloads, pending, blocklist, series, editions, identifiers), nil +} + +func nullID(id int64) any { + if id <= 0 { + return nil + } + return id +} + +// IgnorePending marks pending rows ignored, selected by id or by author +// folder (exactly one of the two). Rows in any other state are left alone. +func (r *UnmatchedUnitRepo) IgnorePending(ctx context.Context, ids []int64, authorFolder string) (int64, error) { + now := unitTime(time.Now()) + var out sql.Result + var err error + switch { + case len(ids) > 0: + ph := make([]string, len(ids)) + args := make([]any, 0, len(ids)+1) + args = append(args, now) + for i, id := range ids { + ph[i] = "?" + args = append(args, id) + } + //nolint:gosec // G202: the IN list is generated ? placeholders; every id is bound + out, err = r.db.ExecContext(ctx, + `UPDATE unmatched_units SET state = 'ignored', updated_at = ? WHERE state = 'pending' AND id IN (`+strings.Join(ph, ",")+`)`, + args...) + case authorFolder != "": + out, err = r.db.ExecContext(ctx, + `UPDATE unmatched_units SET state = 'ignored', updated_at = ? WHERE state = 'pending' AND author_folder = ?`, + now, authorFolder) + default: + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("unmatched units: ignore: %w", err) + } + return out.RowsAffected() +} + +// BookReferencedElsewhere reports whether any row other than exceptID still +// points at bookID, as its adopted book or as a book it created. +func (r *UnmatchedUnitRepo) BookReferencedElsewhere(ctx context.Context, bookID, exceptID int64) (bool, error) { + var one int + err := r.db.QueryRowContext(ctx, + `SELECT 1 FROM unmatched_units WHERE id <> ? AND (book_id = ? OR created_book_id = ?) LIMIT 1`, + exceptID, bookID, bookID).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +// AuthorReferencedElsewhere is BookReferencedElsewhere for a created author. +func (r *UnmatchedUnitRepo) AuthorReferencedElsewhere(ctx context.Context, authorID, exceptID int64) (bool, error) { + var one int + err := r.db.QueryRowContext(ctx, + `SELECT 1 FROM unmatched_units WHERE id <> ? AND created_author_id = ? LIMIT 1`, + exceptID, authorID).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} diff --git a/internal/db/unmatched_units_list.go b/internal/db/unmatched_units_list.go new file mode 100644 index 000000000..5c6c70fdd --- /dev/null +++ b/internal/db/unmatched_units_list.go @@ -0,0 +1,269 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/vavallee/bindery/internal/textutil" +) + +// UnmatchedListQuery filters and orders a page of unmatched units. Every +// field is bound or mapped through a switch; none reaches SQL text (S9). +type UnmatchedListQuery struct { + // State is pending (default), ignored or adopted. + State string + Reason string + AuthorFolder string + Format string + Search string + // Sort is score (default), title, folder, files, size or seen. + Sort string + // Dir is asc or desc; empty picks the sort's natural direction. + Dir string + Limit int + Offset int +} + +// unmatchedListState maps a requested state to the stored state it lists. +// The in flight states (adopting, undoing) last for one request and are not +// listed, which keeps every list on a single state equality and so in index +// order (P4). A row a crashed request left claimed is released by the next +// scan's reconcile. +func unmatchedListState(state string) string { + switch state { + case UnmatchedStateIgnored, UnmatchedStateAdopted: + return state + default: + return UnmatchedStatePending + } +} + +// unmatchedOrderBy maps a sort key and direction to a fixed ORDER BY clause. +func unmatchedOrderBy(sort, dir string) string { + desc := func(natural bool) string { + switch dir { + case "asc": + return "ASC" + case "desc": + return "DESC" + } + if natural { + return "DESC" + } + return "ASC" + } + switch sort { + case "title": + return "parsed_title " + desc(false) + ", id" + case "folder": + return "author_folder " + desc(false) + ", rel_path " + desc(false) + case "files": + return "file_count " + desc(true) + ", rel_path" + case "size": + return "size_bytes " + desc(true) + ", rel_path" + case "seen": + return "first_seen_at " + desc(true) + ", id" + default: + return "top_score " + desc(true) + ", rel_path" + } +} + +func unmatchedWhere(q UnmatchedListQuery, includeFacetFilters bool) (string, []any) { + where := "state = ?" + args := []any{unmatchedListState(q.State)} + if includeFacetFilters { + if q.Reason != "" { + where += " AND reason = ?" + args = append(args, q.Reason) + } + if q.AuthorFolder != "" { + where += " AND author_folder = ?" + args = append(args, q.AuthorFolder) + } + if q.Format != "" { + where += " AND format = ?" + args = append(args, q.Format) + } + } + // A substring match over at most 20,000 rows (the scan's unit cap), one + // LIKE per folded word. FoldForSearch already drops LIKE's wildcards; the + // escape stays so that never depends on the fold. Acceptable at that size, and + // every other predicate above is index backed (P4). + if folded := textutil.FoldForSearch(q.Search); folded != "" { + for _, tok := range searchTokens(folded) { + where += " AND search_key LIKE ? ESCAPE '\\'" + args = append(args, "%"+escapeLike(tok)+"%") + } + } + return where, args +} + +// List returns one page of units and the total matching the filters. +func (r *UnmatchedUnitRepo) List(ctx context.Context, q UnmatchedListQuery) ([]UnmatchedUnit, int, error) { + where, args := unmatchedWhere(q, true) + var total int + if err := r.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM unmatched_units WHERE `+where, args...).Scan(&total); err != nil { + return nil, 0, fmt.Errorf("unmatched units: count: %w", err) + } + limit := q.Limit + if limit <= 0 || limit > 250 { + limit = 50 + } + offset := max(q.Offset, 0) + //nolint:gosec // G202: where is fixed predicates with bound values (unmatchedWhere) and the ORDER BY comes from the unmatchedOrderBy switch + rows, err := r.db.QueryContext(ctx, + `SELECT `+unmatchedUnitColumns+` FROM unmatched_units WHERE `+where+ + ` ORDER BY `+unmatchedOrderBy(q.Sort, q.Dir)+` LIMIT ? OFFSET ?`, + append(args, limit, offset)...) + if err != nil { + return nil, 0, fmt.Errorf("unmatched units: list: %w", err) + } + defer rows.Close() + out := make([]UnmatchedUnit, 0, limit) + for rows.Next() { + u, err := scanUnmatchedUnit(rows.Scan) + if err != nil { + return nil, 0, fmt.Errorf("unmatched units: list scan: %w", err) + } + out = append(out, *u) + } + return out, total, rows.Err() +} + +// FacetCount is one value of a facet and how many units carry it. +type FacetCount struct { + Value string `json:"value"` + Count int `json:"count"` +} + +// FolderFacet is one author folder in the structure rail. +type FolderFacet struct { + Folder string `json:"folder"` + Units int `json:"units"` + Files int `json:"files"` + // NotInLibrary counts units whose author matched no library author. When + // it equals Units the whole folder is one decision: add the author. + NotInLibrary int `json:"notInLibrary"` + // Author is the parsed author most units in the folder share. + Author string `json:"author"` +} + +// UnmatchedFacets are the grouped counts behind the filters and the rail. +type UnmatchedFacets struct { + Reasons []FacetCount `json:"reasons"` + Formats []FacetCount `json:"formats"` + Folders []FolderFacet `json:"folders"` +} + +// maxFolderFacets bounds the rail. The rail is for the big decisions; a long +// tail of one book folders is what the table and its folder filter are for. +const maxFolderFacets = 12 + +// Facets runs the three grouped counts for a state and search. The web asks +// for them only when that pair changes, not on every page turn (P4). +func (r *UnmatchedUnitRepo) Facets(ctx context.Context, q UnmatchedListQuery) (UnmatchedFacets, error) { + where, args := unmatchedWhere(q, false) + f := UnmatchedFacets{Reasons: []FacetCount{}, Formats: []FacetCount{}, Folders: []FolderFacet{}} + grouped := func(col string) ([]FacetCount, error) { + //nolint:gosec // G202: col is one of two literal column names below; where binds every value + rows, err := r.db.QueryContext(ctx, + `SELECT `+col+`, COUNT(*) FROM unmatched_units WHERE `+where+` GROUP BY `+col+` ORDER BY COUNT(*) DESC, `+col, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []FacetCount{} + for rows.Next() { + var fc FacetCount + if err := rows.Scan(&fc.Value, &fc.Count); err != nil { + return nil, err + } + out = append(out, fc) + } + return out, rows.Err() + } + var err error + if f.Reasons, err = grouped("reason"); err != nil { + return f, fmt.Errorf("unmatched units: reason facet: %w", err) + } + if f.Formats, err = grouped("format"); err != nil { + return f, fmt.Errorf("unmatched units: format facet: %w", err) + } + // The most common parsed author per folder comes from a correlated + // subquery that only runs for the handful of folders the LIMIT keeps. Its + // unqualified columns resolve to the innermost table, a. + //nolint:gosec // G202: where is fixed predicates with bound values (unmatchedWhere) + rows, err := r.db.QueryContext(ctx, ` + SELECT g.author_folder, g.units, g.files, g.not_in_library, + COALESCE((SELECT parsed_author FROM unmatched_units a + WHERE a.author_folder = g.author_folder AND a.parsed_author <> '' AND `+where+` + GROUP BY parsed_author ORDER BY COUNT(*) DESC LIMIT 1), '') + FROM ( + SELECT author_folder, COUNT(*) AS units, SUM(file_count) AS files, + SUM(CASE WHEN reason = 'author_not_in_library' THEN 1 ELSE 0 END) AS not_in_library + FROM unmatched_units WHERE author_folder <> '' AND `+where+` + GROUP BY author_folder ORDER BY units DESC, author_folder LIMIT ? + ) g`, append(append(append([]any{}, args...), args...), maxFolderFacets)...) + if err != nil { + return f, fmt.Errorf("unmatched units: folder facet: %w", err) + } + defer rows.Close() + for rows.Next() { + var ff FolderFacet + if err := rows.Scan(&ff.Folder, &ff.Units, &ff.Files, &ff.NotInLibrary, &ff.Author); err != nil { + return f, fmt.Errorf("unmatched units: folder facet scan: %w", err) + } + f.Folders = append(f.Folders, ff) + } + return f, rows.Err() +} + +// UnmatchedBookRef is the slice of a book the adoption list shows beside a +// suggestion or an adopted row. +type UnmatchedBookRef struct { + ID int64 `json:"id"` + Title string `json:"title"` + AuthorID int64 `json:"authorId"` + AuthorName string `json:"authorName"` + ImageURL string `json:"imageUrl,omitempty"` + Status string `json:"status"` + MediaType string `json:"mediaType"` + Monitored bool `json:"monitored"` +} + +// BookRefs loads the books a page of units points at in one IN query (P4, +// T5), whatever the number of rows and candidates on the page. +func (r *UnmatchedUnitRepo) BookRefs(ctx context.Context, ids []int64) (map[int64]UnmatchedBookRef, error) { + out := make(map[int64]UnmatchedBookRef, len(ids)) + if len(ids) == 0 { + return out, nil + } + ph := make([]string, len(ids)) + args := make([]any, len(ids)) + for i, id := range ids { + ph[i] = "?" + args[i] = id + } + //nolint:gosec // G202: the IN list is generated ? placeholders; every id is bound + rows, err := r.db.QueryContext(ctx, ` + SELECT b.id, b.title, b.author_id, COALESCE(a.name, ''), COALESCE(b.image_url, ''), + b.status, b.media_type, b.monitored + FROM books b LEFT JOIN authors a ON a.id = b.author_id + WHERE b.id IN (`+strings.Join(ph, ",")+`)`, args...) + if err != nil { + return nil, fmt.Errorf("unmatched units: book refs: %w", err) + } + defer rows.Close() + for rows.Next() { + var b UnmatchedBookRef + var authorID sql.NullInt64 + if err := rows.Scan(&b.ID, &b.Title, &authorID, &b.AuthorName, &b.ImageURL, &b.Status, &b.MediaType, &b.Monitored); err != nil { + return nil, fmt.Errorf("unmatched units: book refs scan: %w", err) + } + b.AuthorID = authorID.Int64 + out[b.ID] = b + } + return out, rows.Err() +} diff --git a/internal/db/unmatched_units_purge_test.go b/internal/db/unmatched_units_purge_test.go new file mode 100644 index 000000000..320e73b31 --- /dev/null +++ b/internal/db/unmatched_units_purge_test.go @@ -0,0 +1,139 @@ +package db + +import ( + "context" + "testing" + "time" + + "github.com/vavallee/bindery/internal/models" +) + +// TestReconcileScan_KeepsAdoptionsWhileTheirBookExists: an adopted unit's +// files are tracked, so no scan sees it again and its last_seen_at only ages. +// It must stay (Undo stays available) while its book exists, and go once the +// book is deleted. +func TestReconcileScan_KeepsAdoptionsWhileTheirBookExists(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + authors := NewAuthorRepo(database) + books := NewBookRepo(database) + author := &models.Author{ForeignID: "ol:a", Name: "Ann Leckie", SortName: "Leckie, Ann", MetadataProvider: "openlibrary"} + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + book := &models.Book{ForeignID: "ol:b", AuthorID: author.ID, Title: "Provenance", + Status: models.BookStatusWanted, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := books.Create(ctx, book); err != nil { + t.Fatal(err) + } + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/p.epub")}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + u := unitByPath(t, database, repo, "/lib/A/p.epub") + token, _ := repo.Claim(ctx, u.ID, UnmatchedStatePending, UnmatchedStateAdopting) + if token == "" { + t.Fatal("claim failed") + } + if ok, err := repo.CompleteAdoption(ctx, u.ID, token, AdoptionRecord{BookID: book.ID}); err != nil || !ok { + t.Fatalf("complete: %v %v", ok, err) + } + if _, err := database.Exec(`UPDATE unmatched_units SET last_seen_at = ?`, unitTime(time.Now().Add(-90*24*time.Hour))); err != nil { + t.Fatal(err) + } + + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/other.epub")}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + if got := unitByPath(t, database, repo, "/lib/A/p.epub"); got == nil || got.State != UnmatchedStateAdopted { + t.Fatalf("adoption of an existing book purged after 90 days: %+v", got) + } + + if err := books.Delete(ctx, book.ID); err != nil { + t.Fatal(err) + } + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/other.epub")}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + if got := unitByPath(t, database, repo, "/lib/A/p.epub"); got != nil { + t.Fatalf("adoption whose book is gone kept: %+v", got) + } +} + +// TestReconcileScan_PurgesIgnoresUnderARemovedRoot: an old ignored row under a +// root that is no longer configured at all is purged, since no scan will see +// its files again; one under a configured root that found no files this time +// (an unmounted volume) is kept. +func TestReconcileScan_PurgesIgnoresUnderARemovedRoot(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + removed := scanUnit("/old-library/A/gone.epub") + removed.RootPath = "/old-library" + unmounted := scanUnit("/audiobooks/A/away.epub") + unmounted.RootPath = "/audiobooks" + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{removed, unmounted}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`UPDATE unmatched_units SET state = 'ignored', last_seen_at = ?`, + unitTime(time.Now().Add(-40*24*time.Hour))); err != nil { + t.Fatal(err) + } + res, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/new.epub")}, ReconcileScanOptions{ + RootsWithFiles: []string{"/lib"}, ConfiguredRoots: []string{"/lib", "/audiobooks"}, + }) + if err != nil { + t.Fatal(err) + } + if res.Purged != 1 { + t.Fatalf("purged = %d, want only the row under the removed root", res.Purged) + } + if unitByPath(t, database, repo, "/old-library/A/gone.epub") != nil { + t.Fatal("ignored row under a removed root kept") + } + if unitByPath(t, database, repo, "/audiobooks/A/away.epub") == nil { + t.Fatal("ignored row under a configured root that found no files purged") + } +} + +// TestClaimToken_ScopesEveryWrite is the interleaving of two requests on one +// row: A claims it and stalls, recovery releases A's claim, B claims the row. +// Nothing A writes afterwards (progress, completion, reset) may land on B's +// claim. +func TestClaimToken_ScopesEveryWrite(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + book := seedAdoptionBook(t, database) + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/x.epub")}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + id := unitByPath(t, database, repo, "/lib/A/x.epub").ID + + tokenA, _ := repo.Claim(ctx, id, UnmatchedStatePending, UnmatchedStateAdopting) + if ok, _ := repo.ResetToPending(ctx, id, UnmatchedStateAdopting, tokenA); !ok { + t.Fatal("recovery could not release A's claim") + } + tokenB, _ := repo.Claim(ctx, id, UnmatchedStatePending, UnmatchedStateAdopting) + if tokenA == "" || tokenB == "" || tokenA == tokenB { + t.Fatalf("tokens %q %q", tokenA, tokenB) + } + + recA := AdoptionRecord{BookID: book.ID, Registered: []RegisteredFile{{Path: "/lib/A/x.epub", BookID: book.ID}}} + if ok, _ := repo.RecordAdoptionProgress(ctx, id, tokenA, recA); ok { + t.Error("A's progress landed on B's claim") + } + if ok, _ := repo.CompleteAdoption(ctx, id, tokenA, recA); ok { + t.Error("A completed B's claim") + } + if ok, _ := repo.ResetToPending(ctx, id, UnmatchedStateAdopting, tokenA); ok { + t.Error("A's failure path reset B's claim") + } + if ok, _ := repo.ReleaseClaim(ctx, id, UnmatchedStateAdopting, UnmatchedStatePending, tokenA); ok { + t.Error("A released B's claim") + } + u, _ := repo.Get(ctx, id) + if u.State != UnmatchedStateAdopting || u.ClaimToken != tokenB || len(u.Registered) != 0 { + t.Fatalf("row after A's late writes = %+v, want B's untouched claim", u) + } + if ok, err := repo.CompleteAdoption(ctx, id, tokenB, AdoptionRecord{BookID: book.ID}); err != nil || !ok { + t.Fatalf("B's completion = %v %v", ok, err) + } +} diff --git a/internal/db/unmatched_units_test.go b/internal/db/unmatched_units_test.go new file mode 100644 index 000000000..1a2a6c297 --- /dev/null +++ b/internal/db/unmatched_units_test.go @@ -0,0 +1,536 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/vavallee/bindery/internal/models" +) + +func openUnmatchedRepo(t testing.TB) (*sql.DB, *UnmatchedUnitRepo) { + t.Helper() + database, err := OpenMemory() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = database.Close() }) + return database, NewUnmatchedUnitRepo(database) +} + +func scanUnit(path string) UnmatchedUnitScan { + return UnmatchedUnitScan{ + UnitPath: path, UnitKind: UnmatchedKindFile, Format: "ebook", FileCount: 1, SizeBytes: 10, + RootPath: "/lib", RelPath: strings.TrimPrefix(path, "/lib/"), AuthorFolder: "A", + ParsedTitle: "T " + path, ParsedAuthor: "A", Reason: "no_title_match", + MemberPaths: []string{path}, + } +} + +func unitByPath(t *testing.T, database *sql.DB, repo *UnmatchedUnitRepo, path string) *UnmatchedUnit { + t.Helper() + var id int64 + err := database.QueryRow(`SELECT id FROM unmatched_units WHERE unit_path = ?`, path).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + t.Fatal(err) + } + u, err := repo.Get(context.Background(), id) + if err != nil { + t.Fatal(err) + } + return u +} + +// TestReconcileScan_KeepsDecisions covers the ReconcileScan contract: a rescan +// never moves an ignored row, a unit the scan stops seeing is removed while +// pending and kept once decided, and the counts come back. +func TestReconcileScan_KeepsDecisions(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + + first := []UnmatchedUnitScan{scanUnit("/lib/A/one.epub"), scanUnit("/lib/A/two.epub"), scanUnit("/lib/A/three.epub")} + res, err := repo.ReconcileScan(ctx, first, ReconcileScanOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Pending != 3 || res.Generation != 1 { + t.Fatalf("first reconcile = %+v, want 3 pending in generation 1", res) + } + + two := unitByPath(t, database, repo, "/lib/A/two.epub") + if ok, err := repo.ClaimState(ctx, two.ID, UnmatchedStatePending, UnmatchedStateIgnored); err != nil || !ok { + t.Fatalf("ignore two: ok=%v err=%v", ok, err) + } + + // The second scan still sees two (ignored) and one, and no longer sees three. + updated := scanUnit("/lib/A/one.epub") + updated.ParsedTitle = "Renamed" + res, err = repo.ReconcileScan(ctx, []UnmatchedUnitScan{updated, scanUnit("/lib/A/two.epub")}, ReconcileScanOptions{}) + if err != nil { + t.Fatal(err) + } + if res.RemovedPending != 1 || res.Pending != 1 || res.Ignored != 1 { + t.Fatalf("second reconcile = %+v, want 1 removed, 1 pending, 1 ignored", res) + } + if got := unitByPath(t, database, repo, "/lib/A/two.epub"); got == nil || got.State != UnmatchedStateIgnored { + t.Fatalf("ignored row after rescan = %+v, want still ignored", got) + } + if got := unitByPath(t, database, repo, "/lib/A/three.epub"); got != nil { + t.Fatalf("vanished pending row survived: %+v", got) + } + if got := unitByPath(t, database, repo, "/lib/A/one.epub"); got.ParsedTitle != "Renamed" || got.ID == two.ID { + t.Fatalf("pending row not refreshed in place: %+v", got) + } +} + +// TestReconcileScan_ZeroFileScanDeletesNothing is the unmounted volume case: a +// scan that found nothing must not erase a single row or decision. +func TestReconcileScan_ZeroFileScanDeletesNothing(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/one.epub"), scanUnit("/lib/A/two.epub")}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + two := unitByPath(t, database, repo, "/lib/A/two.epub") + if _, err := repo.ClaimState(ctx, two.ID, UnmatchedStatePending, UnmatchedStateIgnored); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-90 * 24 * time.Hour) + if _, err := database.Exec(`UPDATE unmatched_units SET last_seen_at = ?`, unitTime(old)); err != nil { + t.Fatal(err) + } + res, err := repo.ReconcileScan(ctx, nil, ReconcileScanOptions{SkipDeletion: true}) + if err != nil { + t.Fatal(err) + } + if res.RemovedPending != 0 || res.Purged != 0 || res.Pending != 1 || res.Ignored != 1 { + t.Fatalf("zero file reconcile = %+v, want nothing removed", res) + } +} + +// TestReconcileScan_PurgesOldDecisions: ignored rows unseen for 30 days go +// when their root produced files, a truncated scan removes and purges +// nothing, a root that produced no files keeps its ignores, and a recent one +// stays. +func TestReconcileScan_PurgesOldDecisions(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + units := []UnmatchedUnitScan{scanUnit("/lib/A/old.epub"), scanUnit("/lib/A/recent.epub")} + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`UPDATE unmatched_units SET state = 'ignored'`); err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`UPDATE unmatched_units SET last_seen_at = ? WHERE unit_path = '/lib/A/old.epub'`, + unitTime(time.Now().Add(-31*24*time.Hour))); err != nil { + t.Fatal(err) + } + + res, err := repo.ReconcileScan(ctx, nil, ReconcileScanOptions{SkipDeletion: true, RootsWithFiles: []string{"/lib"}}) + if err != nil { + t.Fatal(err) + } + if res.Purged != 0 || res.Ignored != 2 { + t.Fatalf("truncated reconcile = %+v, want no purge", res) + } + res, err = repo.ReconcileScan(ctx, nil, ReconcileScanOptions{RootsWithFiles: []string{"/audiobooks"}}) + if err != nil { + t.Fatal(err) + } + if res.Purged != 0 || res.Ignored != 2 { + t.Fatalf("reconcile with the row's root producing no files = %+v, want no purge", res) + } + res, err = repo.ReconcileScan(ctx, nil, ReconcileScanOptions{RootsWithFiles: []string{"/lib"}}) + if err != nil { + t.Fatal(err) + } + if res.Purged != 1 || res.Ignored != 1 { + t.Fatalf("reconcile = %+v, want the 31 day old row purged", res) + } + if unitByPath(t, database, repo, "/lib/A/recent.epub") == nil { + t.Fatal("recent ignored row purged") + } +} + +func seedAdoptionBook(t *testing.T, database *sql.DB) *models.Book { + t.Helper() + ctx := context.Background() + author := &models.Author{ForeignID: "ol:seed", Name: "Seed Author", SortName: "Author, Seed", MetadataProvider: "openlibrary"} + if err := NewAuthorRepo(database).Create(ctx, author); err != nil { + t.Fatal(err) + } + book := &models.Book{ForeignID: "ol:seed-book", AuthorID: author.ID, Title: "Seed", + Status: models.BookStatusWanted, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := NewBookRepo(database).Create(ctx, book); err != nil { + t.Fatal(err) + } + return book +} + +// TestReconcileScan_AdoptedRowSeenAgain: an adoption finished before the scan +// started whose unit is unmatched again was undone elsewhere, so it returns to +// pending; one finished while the scan ran is left alone. +func TestReconcileScan_AdoptedRowSeenAgain(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + units := []UnmatchedUnitScan{scanUnit("/lib/A/before.epub"), scanUnit("/lib/A/during.epub")} + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + book := seedAdoptionBook(t, database) + for _, p := range []string{"/lib/A/before.epub", "/lib/A/during.epub"} { + u := unitByPath(t, database, repo, p) + token, _ := repo.Claim(ctx, u.ID, UnmatchedStatePending, UnmatchedStateAdopting) + if token == "" { + t.Fatal("claim failed") + } + if ok, err := repo.CompleteAdoption(ctx, u.ID, token, AdoptionRecord{BookID: book.ID, Registered: []RegisteredFile{{Path: p, BookID: book.ID}}}); err != nil || !ok { + t.Fatalf("complete: ok=%v err=%v", ok, err) + } + } + scanStart := time.Now() + if _, err := database.Exec(`UPDATE unmatched_units SET resolved_at = ? WHERE unit_path = '/lib/A/before.epub'`, + unitTime(scanStart.Add(-time.Hour))); err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`UPDATE unmatched_units SET resolved_at = ? WHERE unit_path = '/lib/A/during.epub'`, + unitTime(scanStart.Add(time.Minute))); err != nil { + t.Fatal(err) + } + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{StartedAt: scanStart}); err != nil { + t.Fatal(err) + } + if got := unitByPath(t, database, repo, "/lib/A/before.epub"); got.State != UnmatchedStatePending || len(got.Registered) != 0 { + t.Fatalf("stale adoption = %+v, want pending with nothing registered", got) + } + if got := unitByPath(t, database, repo, "/lib/A/during.epub"); got.State != UnmatchedStateAdopted { + t.Fatalf("adoption during the scan = %+v, want still adopted", got) + } +} + +// TestReconcileScan_ChunksAndLeavesClaims: more than one chunk of units +// lands, a truncated scan removes no pending row, and a scan never touches a +// row an adopt holds (its side effects are the adoption handler's to reverse). +func TestReconcileScan_ChunksAndLeavesClaims(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + units := make([]UnmatchedUnitScan, reconcileChunkSize*2+7) + for i := range units { + units[i] = scanUnit(fmt.Sprintf("/lib/A/%04d.epub", i)) + } + res, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Upserted != len(units) || res.Pending != len(units) { + t.Fatalf("reconcile = %+v, want %d rows", res, len(units)) + } + claimed := unitByPath(t, database, repo, "/lib/A/0000.epub") + if ok, _ := repo.ClaimState(ctx, claimed.ID, UnmatchedStatePending, UnmatchedStateAdopting); !ok { + t.Fatal("claim failed") + } + if res, err = repo.ReconcileScan(ctx, units[1:2], ReconcileScanOptions{SkipDeletion: true}); err != nil || res.RemovedPending != 0 { + t.Fatalf("truncated reconcile = %+v %v, want no pending row removed", res, err) + } + if _, err := repo.ReconcileScan(ctx, units[1:], ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + if got := unitByPath(t, database, repo, "/lib/A/0000.epub"); got == nil || got.State != UnmatchedStateAdopting { + t.Fatalf("claimed row after scans = %+v, want untouched", got) + } +} + +// TestStaleClaims_DatedByClaimedAt: a scan refreshes updated_at on every row +// it sees, so a claim is dated by claimed_at. A long held claim is stale even +// when a scan just touched the row; a fresh one is not. +func TestStaleClaims_DatedByClaimedAt(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + units := []UnmatchedUnitScan{scanUnit("/lib/A/old.epub"), scanUnit("/lib/A/new.epub")} + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + for _, p := range []string{"/lib/A/old.epub", "/lib/A/new.epub"} { + if ok, _ := repo.ClaimState(ctx, unitByPath(t, database, repo, p).ID, UnmatchedStatePending, UnmatchedStateAdopting); !ok { + t.Fatal("claim failed") + } + } + if _, err := database.Exec(`UPDATE unmatched_units SET claimed_at = ? WHERE unit_path = '/lib/A/old.epub'`, + unitTime(time.Now().Add(-time.Hour))); err != nil { + t.Fatal(err) + } + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + stale, err := repo.StaleClaims(ctx, time.Now().Add(-UnmatchedClaimTimeout)) + if err != nil { + t.Fatal(err) + } + if len(stale) != 1 || stale[0].UnitPath != "/lib/A/old.epub" { + t.Fatalf("stale claims = %+v, want only the hour old claim", stale) + } + if ok, err := repo.ResetToPending(ctx, stale[0].ID, UnmatchedStateAdopting, stale[0].ClaimToken); err != nil || !ok { + t.Fatalf("reset: %v %v", ok, err) + } + if got := unitByPath(t, database, repo, "/lib/A/old.epub"); got.State != UnmatchedStatePending || got.ClaimedAt != nil { + t.Fatalf("reset row = %+v", got) + } +} + +// TestClaimState_ExactlyOneWinner is T3 at the repository: many goroutines +// racing to claim one row, one wins. +func TestClaimState_ExactlyOneWinner(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/x.epub")}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + u := unitByPath(t, database, repo, "/lib/A/x.epub") + var wg sync.WaitGroup + var mu sync.Mutex + wins := 0 + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + ok, err := repo.ClaimState(ctx, u.ID, UnmatchedStatePending, UnmatchedStateAdopting) + if err != nil { + t.Error(err) + } + if ok { + mu.Lock() + wins++ + mu.Unlock() + } + }() + } + wg.Wait() + if wins != 1 { + t.Fatalf("claims won = %d, want 1", wins) + } +} + +// TestUnmatchedList_FiltersSortsAndEscapes covers the list parameters S9 names: +// an unknown sort falls back, search is folded and bound, and the folder +// filter is bound. +func TestUnmatchedList_FiltersSortsAndEscapes(t *testing.T) { + ctx := context.Background() + _, repo := openUnmatchedRepo(t) + a := scanUnit("/lib/Weir/Hail Mary.epub") + a.ParsedTitle, a.AuthorFolder, a.Candidates = "Project Hail Mary", "Weir", []UnmatchedCandidate{{BookID: 1, Score: 0.7}} + b := scanUnit("/lib/Other/100% Done.epub") + b.ParsedTitle, b.AuthorFolder, b.FileCount = "100% Done", "Other", 3 + c := scanUnit("/lib/Other/100 Done.epub") + c.ParsedTitle, c.AuthorFolder, c.Reason = "100 Done", "Other", "author_not_in_library" + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{a, b, c}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + + items, total, err := repo.List(ctx, UnmatchedListQuery{Sort: "score); DROP TABLE unmatched_units; --"}) + if err != nil || total != 3 || items[0].ParsedTitle != "Project Hail Mary" { + t.Fatalf("default list = %d items, total %d, err %v", len(items), total, err) + } + items, _, err = repo.List(ctx, UnmatchedListQuery{Search: "hail MARY"}) + if err != nil || len(items) != 1 || items[0].ParsedTitle != "Project Hail Mary" { + t.Fatalf("search = %+v err %v", items, err) + } + // A bare wildcard folds away to no term at all: every row, not an error + // and not a pattern. + if _, total, err = repo.List(ctx, UnmatchedListQuery{Search: "%_"}); err != nil || total != 3 { + t.Fatalf("wildcard search total = %d err %v, want 3", total, err) + } + items, _, err = repo.List(ctx, UnmatchedListQuery{AuthorFolder: "Other' OR '1'='1", Sort: "files"}) + if err != nil || len(items) != 0 { + t.Fatalf("injected folder matched %d rows", len(items)) + } + items, _, err = repo.List(ctx, UnmatchedListQuery{AuthorFolder: "Other", Sort: "files"}) + if err != nil || len(items) != 2 || items[0].FileCount != 3 { + t.Fatalf("folder list sorted by files = %+v err %v", items, err) + } + + f, err := repo.Facets(ctx, UnmatchedListQuery{}) + if err != nil { + t.Fatal(err) + } + if len(f.Folders) != 2 || f.Folders[0].Folder != "Other" || f.Folders[0].Units != 2 || f.Folders[0].Files != 4 || f.Folders[0].NotInLibrary != 1 { + t.Fatalf("folder facets = %+v", f.Folders) + } + if len(f.Reasons) != 2 || len(f.Formats) != 1 { + t.Fatalf("facets = %+v", f) + } +} + +func queryPlan(t *testing.T, database *sql.DB, query string, args ...any) string { + t.Helper() + rows, err := database.Query(`EXPLAIN QUERY PLAN `+query, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return strings.Join(plan, " | ") +} + +// TestUnmatchedList_QueryPlansUseIndexes is P4: the default list and every sort +// search an index for the state filter instead of scanning the table. +func TestUnmatchedList_QueryPlansUseIndexes(t *testing.T) { + database, _ := openUnmatchedRepo(t) + for _, sort := range []string{"", "score", "title", "folder", "files", "size", "seen"} { + q := UnmatchedListQuery{Sort: sort} + where, args := unmatchedWhere(q, true) + joined := queryPlan(t, database, `SELECT id FROM unmatched_units WHERE `+where+ + ` ORDER BY `+unmatchedOrderBy(q.Sort, q.Dir)+` LIMIT 50`, args...) + if !strings.Contains(joined, "INDEX idx_unmatched_units_state") || strings.Contains(joined, "SCAN unmatched_units") { + t.Errorf("sort %q plan does not search a state index: %s", sort, joined) + } + if (sort == "" || sort == "score" || sort == "folder") && strings.Contains(joined, "TEMP B-TREE") { + t.Errorf("sort %q should be served in index order: %s", sort, joined) + } + } +} + +// TestBookRefs_OneQueryPerPage hydrates many ids through a single statement. +func TestBookRefs_OneQueryPerPage(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + authors := NewAuthorRepo(database) + books := NewBookRepo(database) + author := &models.Author{ForeignID: "ol:a", Name: "Ann Leckie", SortName: "Leckie, Ann", MetadataProvider: "openlibrary"} + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + var ids []int64 + for i := range 5 { + b := &models.Book{ForeignID: fmt.Sprintf("ol:b%d", i), AuthorID: author.ID, Title: fmt.Sprintf("Book %d", i), + Status: models.BookStatusWanted, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := books.Create(ctx, b); err != nil { + t.Fatal(err) + } + ids = append(ids, b.ID) + } + refs, err := repo.BookRefs(ctx, append(ids, 99999)) + if err != nil { + t.Fatal(err) + } + if len(refs) != 5 || refs[ids[2]].AuthorName != "Ann Leckie" || refs[ids[2]].Title != "Book 2" { + t.Fatalf("refs = %+v", refs) + } +} + +// TestMigrate088_OverPopulatedDatabase is T4: 088 applied to a database that +// already holds authors, books, notifications and users adds an empty table, +// leaves every existing row alone, and clears a book reference on delete +// instead of blocking it. +func TestMigrate088_OverPopulatedDatabase(t *testing.T) { + ctx := context.Background() + database, err := OpenMemory() + if err != nil { + t.Fatal(err) + } + defer database.Close() + + for _, stmt := range []string{ + `DROP INDEX idx_unmatched_units_book`, `DROP INDEX idx_unmatched_units_state_reason`, + `DROP INDEX idx_unmatched_units_state_folder`, `DROP INDEX idx_unmatched_units_state_score`, + `DROP TABLE unmatched_units`, + } { + if _, err := database.Exec(stmt); err != nil { + t.Fatalf("%s: %v", stmt, err) + } + } + + authors := NewAuthorRepo(database) + books := NewBookRepo(database) + author := &models.Author{ForeignID: "ol:a", Name: "Ann Leckie", SortName: "Leckie, Ann", MetadataProvider: "openlibrary"} + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + book := &models.Book{ForeignID: "ol:b", AuthorID: author.ID, Title: "Ancillary Justice", + Status: models.BookStatusWanted, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := books.Create(ctx, book); err != nil { + t.Fatal(err) + } + if err := NewNotificationRepo(database).Create(ctx, &models.Notification{Name: "hook", Type: "webhook", URL: "http://example.invalid", Enabled: true}); err != nil { + t.Fatal(err) + } + if _, err := NewUserRepo(database).Create(ctx, "reader", "hash"); err != nil { + t.Fatal(err) + } + + v := migrationVersionForTest(t, "088_unmatched_units.sql") + if _, err := database.Exec(`DELETE FROM schema_migrations WHERE version = ?`, v); err != nil { + t.Fatal(err) + } + if err := migrate(database); err != nil { + t.Fatalf("rerun migration 088: %v", err) + } + + var n int + if err := database.QueryRow(`SELECT COUNT(*) FROM unmatched_units`).Scan(&n); err != nil || n != 0 { + t.Fatalf("unmatched_units after migration: n=%d err=%v", n, err) + } + for table, want := range map[string]int{"authors": 1, "books": 1, "notifications": 1, "users": 1} { + if err := database.QueryRow(`SELECT COUNT(*) FROM ` + table).Scan(&n); err != nil || n != want { + t.Errorf("%s rows after migration = %d (err %v), want %d", table, n, err, want) + } + } + + repo := NewUnmatchedUnitRepo(database) + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/x.epub")}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + u := unitByPath(t, database, repo, "/lib/A/x.epub") + if u.State != UnmatchedStatePending { + t.Fatalf("new row state = %q, want pending by default", u.State) + } + token, err := repo.Claim(ctx, u.ID, UnmatchedStatePending, UnmatchedStateAdopting) + if err != nil { + t.Fatal(err) + } + if _, err := repo.CompleteAdoption(ctx, u.ID, token, AdoptionRecord{BookID: book.ID, CreatedBookID: book.ID, CreatedAuthorID: author.ID}); err != nil { + t.Fatal(err) + } + if err := books.Delete(ctx, book.ID); err != nil { + t.Fatalf("delete a book an adoption points at: %v", err) + } + if got := unitByPath(t, database, repo, "/lib/A/x.epub"); got.BookID != 0 || got.CreatedBookID != 0 { + t.Fatalf("book references after delete = %+v, want cleared", got) + } +} + +// BenchmarkReconcileScan is T6: one full reconcile at the scan's unit cap. +func BenchmarkReconcileScan(b *testing.B) { + ctx := context.Background() + units := make([]UnmatchedUnitScan, 20000) + for i := range units { + units[i] = scanUnit(fmt.Sprintf("/lib/Author %03d/Book %05d/book.epub", i%400, i)) + units[i].Candidates = []UnmatchedCandidate{{BookID: int64(i), Score: 0.7}} + } + _, repo := openUnmatchedRepo(b) + b.ResetTimer() + for b.Loop() { + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/importer/scanner.go b/internal/importer/scanner.go index f81c86b27..471771023 100644 --- a/internal/importer/scanner.go +++ b/internal/importer/scanner.go @@ -101,6 +101,9 @@ type Scanner struct { absLib absNotifier absLibraryIDsFn func() []string notif eventNotifier + // unmatchedUnits stores the books a library scan could not match, for + // library adoption. Nil stores nothing (see WithUnmatchedUnits). + unmatchedUnits UnmatchedUnitStore // jobs, when set, tracks the detached scan goroutine launched by StartScan // so process shutdown can cancel and drain it before the database closes @@ -2852,9 +2855,13 @@ func (s *Scanner) scanLibrary(ctx context.Context) { return } + scanStartedAt := time.Now() + // walkDir appends all book files found under root to foundFiles, tracking // which root each file belongs to so the author-inference fallback can - // strip the correct prefix. + // strip the correct prefix. The size and mode the walk already has are + // kept for the unmatched units (P2). + walked := make(map[string]walkedFile) walkDir := func(root string) []string { var files []string if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { @@ -2863,6 +2870,7 @@ func (s *Scanner) scanLibrary(ctx context.Context) { } if IsBookFile(path) { files = append(files, path) + walked[path] = walkedFile{size: info.Size(), mode: info.Mode()} } return nil }); err != nil { @@ -2872,16 +2880,26 @@ func (s *Scanner) scanLibrary(ctx context.Context) { } foundFiles := walkDir(s.libraryDir) + // rootsWithFiles tells the unmatched unit purge which roots this scan + // actually saw files under, so an unmounted root keeps its ignores. + var rootsWithFiles []string + if len(foundFiles) > 0 { + rootsWithFiles = append(rootsWithFiles, s.libraryDir) + } // Also scan the audiobook directory when it is configured separately. if s.audiobookDir != "" && s.audiobookDir != s.libraryDir { - foundFiles = append(foundFiles, walkDir(s.audiobookDir)...) + audioFiles := walkDir(s.audiobookDir) + if len(audioFiles) > 0 { + rootsWithFiles = append(rootsWithFiles, s.audiobookDir) + } + foundFiles = append(foundFiles, audioFiles...) } slog.Info("library scan found files", "paths", []string{s.libraryDir, s.audiobookDir}, "count", len(foundFiles)) if len(foundFiles) == 0 { - s.writeScanResult(ctx, len(foundFiles), 0, 0, 0, 0, nil) + s.writeScanResult(ctx, len(foundFiles), 0, 0, 0, 0, s.unmatchedUnitCounts(ctx, false)) return } @@ -3130,7 +3148,7 @@ func (s *Scanner) scanLibrary(ctx context.Context) { return set, parsedAuthor } - var unmatchedFiles []unmatchedFile + var unmatchedFiles unmatchedCollector var reconciled, unmatched, alreadyTracked, tagReadFailed int // tryReconcileTitle attempts to reconcile path to the given wanted book via @@ -3460,15 +3478,10 @@ func (s *Scanner) scanLibrary(ctx context.Context) { slog.Debug("library scan: unmatched file", "path", path, "parsedTitle", parsed.Title, "parsedAuthor", parsed.Author, "matchAuthor", matchAuthor, "reason", reason) unmatched++ - // Collect up to 1000 unmatched entries for UI display - if len(unmatchedFiles) < 1000 { - unmatchedFiles = append(unmatchedFiles, unmatchedFile{ - Path: path, - ParsedTitle: parsed.Title, - ParsedAuthor: parsed.Author, - Reason: reason, - }) - } + unmatchedFiles.add(unmatchedScanFile{ + path: path, format: detectedFmt, size: walked[path].size, mode: walked[path].mode, + title: parsed.Title, author: parsed.Author, layoutAuthor: layoutAuthor, reason: reason, + }) } } @@ -3488,7 +3501,15 @@ func (s *Scanner) scanLibrary(ctx context.Context) { slog.Info("library scan complete", "paths", scanRoots, "bookFiles", len(foundFiles), "reconciled", reconciled, "unmatched", unmatched, "tagReadFailed", tagReadFailed) - s.writeScanResult(ctx, len(foundFiles), reconciled, unmatched, alreadyTracked, tagReadFailed, unmatchedFiles) + // Suggestions come from the catalogue already in memory, ranked once per + // unit rather than per file. + units := s.recordUnmatchedUnits(ctx, &unmatchedFiles, scanRoots, rootsWithFiles, scanStartedAt, + func(title, author, layoutAuthor string) []db.UnmatchedCandidate { + authorSet, _ := resolveAuthors(author, layoutAuthor) + return rankCandidates(normalizeTitle(title), wantedBooks, booksByAuthor, authorSet) + }) + + s.writeScanResult(ctx, len(foundFiles), reconciled, unmatched, alreadyTracked, tagReadFailed, units) } // refreshStaleRenderedPaths re-derives the file path shown for books that @@ -3640,22 +3661,12 @@ const ( unmatchedReasonNoTitleParsed = "no_title_parsed" ) -// unmatchedFile represents a file that could not be reconciled during library scan. -type unmatchedFile struct { - Path string `json:"path"` - ParsedTitle string `json:"parsed_title"` - ParsedAuthor string `json:"parsed_author"` - // Reason is one of the unmatchedReason* constants. Omitted when empty so - // results written before this field existed keep parsing unchanged. - Reason string `json:"reason,omitempty"` -} - // writeScanError persists a failed-scan result so the UI reflects the failure // instead of a stale prior scan (#965). It reuses the same "library.lastScan" // shape as a normal scan — zero counts plus a non-empty scan_error message the // frontend renders the same way as the other scan-outcome warnings. func (s *Scanner) writeScanError(ctx context.Context, message string) { - s.writeScanResultWithError(ctx, 0, 0, 0, 0, 0, nil, message) + s.writeScanResultWithError(ctx, 0, 0, 0, 0, 0, s.unmatchedUnitCounts(ctx, false), message) } // writeScanResult persists the scan summary to the settings table under @@ -3664,32 +3675,18 @@ func (s *Scanner) writeScanError(ctx context.Context, message string) { // api.SettingLibraryLastScan on the read side, where the settings endpoints // reserve it for admins because the blob carries absolute paths (#2361). Keep // the two spellings in sync. -func (s *Scanner) writeScanResult(ctx context.Context, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed int, unmatchedFiles []unmatchedFile) { - s.writeScanResultWithError(ctx, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed, unmatchedFiles, "") +func (s *Scanner) writeScanResult(ctx context.Context, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed int, units unitCounts) { + s.writeScanResultWithError(ctx, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed, units, "") } // writeScanResultWithError is the shared writer for both successful scans and // early-return failures. scanError is empty for a normal scan and a // user-facing message when the scan could not complete (#965). -func (s *Scanner) writeScanResultWithError(ctx context.Context, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed int, unmatchedFiles []unmatchedFile, scanError string) { +func (s *Scanner) writeScanResultWithError(ctx context.Context, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed int, units unitCounts, scanError string) { if s.settings == nil { return } - // Marshal unmatched files to JSON - var unmatchedJSON string - if len(unmatchedFiles) > 0 { - bytes, err := json.Marshal(unmatchedFiles) - if err != nil { - slog.Warn("library scan: failed to marshal unmatched files", "error", err) - unmatchedJSON = "[]" - } else { - unmatchedJSON = string(bytes) - } - } else { - unmatchedJSON = "[]" - } - // Surface the resolved roots that were actually walked so the UI can tell // the user WHICH paths produced the file count. The scanner unions the // audiobook root with the library root only when it differs (see @@ -3719,11 +3716,15 @@ func (s *Scanner) writeScanResultWithError(ctx context.Context, filesFound, reco } } + // The unmatched files now live in unmatched_units, one row per book, and + // the blob carries only their counts. unmatched_files stays as an empty + // list so a web bundle cached from before library adoption still parses + // the result. Added 2026-09 for v1.37; remove it in the release after. payload := fmt.Sprintf( - `{"ran_at":%q,"files_found":%d,"reconciled":%d,"unmatched":%d,"already_tracked":%d,"tag_read_failed":%d,"unmatched_files":%s,"library_dir":%q,"audiobook_dir":%q,"scanned_paths":%s,"no_files_found":%t,"scan_error":%s}`, + `{"ran_at":%q,"files_found":%d,"reconciled":%d,"unmatched":%d,"already_tracked":%d,"tag_read_failed":%d,"unmatched_files":[],"unmatched_units":%d,"ignored_units":%d,"units_truncated":%t,"library_dir":%q,"audiobook_dir":%q,"scanned_paths":%s,"no_files_found":%t,"scan_error":%s}`, time.Now().UTC().Format(time.RFC3339), filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed, - unmatchedJSON, + units.pending, units.ignored, units.truncated, s.libraryDir, s.audiobookDir, pathsJSON, noFilesFound, scanErrorJSON, ) if err := s.settings.Set(ctx, "library.lastScan", payload); err != nil { diff --git a/internal/importer/scanner_claim_rank_test.go b/internal/importer/scanner_claim_rank_test.go index b3cbd905a..11fe67b38 100644 --- a/internal/importer/scanner_claim_rank_test.go +++ b/internal/importer/scanner_claim_rank_test.go @@ -137,8 +137,8 @@ func TestScanLibrary_EbookContainerOutranksTextSidecar(t *testing.T) { // The passed-over sidecar is the container's companion, not an // orphan the user has to go and fix, so it must not show up in the // Unmatched list either. - for _, f := range readUnmatchedFiles(t, ctx, settings) { - if filepath.Clean(f.Path) == filepath.Clean(sidecar) { + for _, f := range readUnmatchedFiles(t, ctx, s) { + if filepath.Clean(f.UnitPath) == filepath.Clean(sidecar) { t.Errorf("sidecar reported unmatched with reason %q; it is the epub's companion", f.Reason) } } diff --git a/internal/importer/scanner_unmatched_blob_test.go b/internal/importer/scanner_unmatched_blob_test.go new file mode 100644 index 000000000..c35bf41f5 --- /dev/null +++ b/internal/importer/scanner_unmatched_blob_test.go @@ -0,0 +1,96 @@ +package importer + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/vavallee/bindery/internal/db" +) + +// This file reads only the library.lastScan blob, so it builds and runs +// against the tree from before library adoption too. That is what records +// the fail before evidence for #2547 and the dropped 1000 file cap. + +type unmatchedBlob struct { + FilesFound int `json:"files_found"` + Reconciled int `json:"reconciled"` + Unmatched int `json:"unmatched"` + AlreadyTracked int `json:"already_tracked"` + UnmatchedFiles []json.RawMessage `json:"unmatched_files"` + UnmatchedUnits *int `json:"unmatched_units"` + UnitsTruncated bool `json:"units_truncated"` +} + +func readUnmatchedBlob(t *testing.T, ctx context.Context, settings *db.SettingsRepo) unmatchedBlob { + t.Helper() + setting, err := settings.Get(ctx, "library.lastScan") + if err != nil || setting == nil { + t.Fatalf("library.lastScan: %v %v", setting, err) + } + var b unmatchedBlob + if err := json.Unmarshal([]byte(setting.Value), &b); err != nil { + t.Fatalf("unmarshal scan result: %v", err) + } + return b +} + +// TestScanLibrary_TrackFolderIsOneUnmatchedBook is #2547: a 193 track +// audiobook nobody has in the library is one book to decide about, not 193 +// rows. The file counters keep their #1436 invariant (files_found equals +// reconciled + unmatched + already_tracked) because they count files. +func TestScanLibrary_TrackFolderIsOneUnmatchedBook(t *testing.T) { + s, _, _, _, settings, libraryDir, ctx := unmatchedFixture(t) + dir := filepath.Join(libraryDir, "Brandon Sanderson", "Rhythm of War") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for i := 1; i <= 193; i++ { + track := filepath.Join(dir, fmt.Sprintf("%03d - Rhythm of War.mp3", i)) + if err := os.WriteFile(track, buildID3v23(fmt.Sprintf("%03d-193", i), "Brandon Sanderson", ""), 0o644); err != nil { + t.Fatal(err) + } + } + + s.ScanLibrary(ctx) + + b := readUnmatchedBlob(t, ctx, settings) + if b.FilesFound != 193 || b.Unmatched != 193 || b.FilesFound != b.Reconciled+b.Unmatched+b.AlreadyTracked { + t.Errorf("counters = %+v, want 193 files, all unmatched, invariant intact", b) + } + if len(b.UnmatchedFiles) != 0 { + t.Errorf("unmatched_files has %d per file entries, want none: the list is units now", len(b.UnmatchedFiles)) + } + if b.UnmatchedUnits == nil || *b.UnmatchedUnits != 1 { + t.Errorf("unmatched_units = %v, want 1 book for the 193 tracks", b.UnmatchedUnits) + } +} + +// TestScanLibrary_UnmatchedListHasNoThousandCap: the old list stopped at 1000 +// files and silently dropped the rest. 1200 unmatched books are 1200 units. +func TestScanLibrary_UnmatchedListHasNoThousandCap(t *testing.T) { + s, _, _, _, settings, libraryDir, ctx := unmatchedFixture(t) + dir := filepath.Join(libraryDir, "Unknown Author") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for i := range 1200 { + if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("Book Number %04d.txt", i)), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + s.ScanLibrary(ctx) + + b := readUnmatchedBlob(t, ctx, settings) + if b.Unmatched != 1200 { + t.Fatalf("unmatched = %d, want 1200", b.Unmatched) + } + if b.UnmatchedUnits == nil || *b.UnmatchedUnits != 1200 || b.UnitsTruncated { + t.Errorf("unmatched_units = %v (truncated %v), want all 1200 stored; unmatched_files held %d", + b.UnmatchedUnits, b.UnitsTruncated, len(b.UnmatchedFiles)) + } +} diff --git a/internal/importer/scanner_unmatched_reason_test.go b/internal/importer/scanner_unmatched_reason_test.go index b891354fd..f6cc4c618 100644 --- a/internal/importer/scanner_unmatched_reason_test.go +++ b/internal/importer/scanner_unmatched_reason_test.go @@ -2,7 +2,6 @@ package importer import ( "context" - "encoding/json" "os" "path/filepath" "testing" @@ -11,32 +10,25 @@ import ( "github.com/vavallee/bindery/internal/models" ) -// unmatchedReasonPayload reads back just the unmatched entries of the persisted -// scan result, including the #1958 reason field. -type unmatchedReasonPayload struct { - Unmatched int `json:"unmatched"` - UnmatchedFiles []unmatchedFile `json:"unmatched_files"` -} - -func readUnmatchedFiles(t *testing.T, ctx context.Context, settings *db.SettingsRepo) []unmatchedFile { +// readUnmatchedFiles reads back the unmatched units the scan stored, with the +// #1958 reason each carries. Each test here lays one file per book, so a unit +// is one file. +func readUnmatchedFiles(t *testing.T, ctx context.Context, s *Scanner) []db.UnmatchedUnit { t.Helper() - setting, err := settings.Get(ctx, "library.lastScan") - if err != nil { - t.Fatalf("get library.lastScan: %v", err) + repo, ok := s.unmatchedUnits.(*db.UnmatchedUnitRepo) + if !ok { + t.Fatal("scanner has no unmatched unit repo wired") } - if setting == nil { - t.Fatal("expected library.lastScan to be persisted, got nil") - } - var p unmatchedReasonPayload - if err := json.Unmarshal([]byte(setting.Value), &p); err != nil { - t.Fatalf("unmarshal scan result %q: %v", setting.Value, err) + units, _, err := repo.List(ctx, db.UnmatchedListQuery{Limit: 250}) + if err != nil { + t.Fatalf("list unmatched units: %v", err) } - return p.UnmatchedFiles + return units } // scanOneUnmatched lays a single epub at rel under the library dir, scans, and // returns the one unmatched entry the scan must have produced. -func scanOneUnmatched(t *testing.T, s *Scanner, settings *db.SettingsRepo, ctx context.Context, libraryDir string, rel ...string) unmatchedFile { +func scanOneUnmatched(t *testing.T, s *Scanner, ctx context.Context, libraryDir string, rel ...string) db.UnmatchedUnit { t.Helper() path := filepath.Join(append([]string{libraryDir}, rel...)...) if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { @@ -46,7 +38,7 @@ func scanOneUnmatched(t *testing.T, s *Scanner, settings *db.SettingsRepo, ctx c s.ScanLibrary(ctx) - files := readUnmatchedFiles(t, ctx, settings) + files := readUnmatchedFiles(t, ctx, s) if len(files) != 1 { t.Fatalf("expected exactly one unmatched file, got %d (%+v)", len(files), files) } @@ -59,7 +51,7 @@ func scanOneUnmatched(t *testing.T, s *Scanner, settings *db.SettingsRepo, ctx c // cannot help — there is no such author to refresh — and it pointed away from // the evidence (the parsed author) for two weeks. func TestScanLibrary_UnmatchedReasonAuthorNotInLibrary(t *testing.T) { - s, _, books, authors, settings, libraryDir, ctx := unmatchedFixture(t) + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) author := &models.Author{ForeignID: "ol:weir", Name: "Andy Weir", SortName: "Weir, Andy", Monitored: true, MetadataProvider: "openlibrary"} if err := authors.Create(ctx, author); err != nil { @@ -74,7 +66,7 @@ func TestScanLibrary_UnmatchedReasonAuthorNotInLibrary(t *testing.T) { t.Fatal(err) } - got := scanOneUnmatched(t, s, settings, ctx, libraryDir, "Becky Chambers", "A Psalm for the Wild-Built", "A Psalm for the Wild-Built.epub") + got := scanOneUnmatched(t, s, ctx, libraryDir, "Becky Chambers", "A Psalm for the Wild-Built", "A Psalm for the Wild-Built.epub") if got.Reason != unmatchedReasonAuthorNotInLibrary { t.Errorf("reason = %q, want %q", got.Reason, unmatchedReasonAuthorNotInLibrary) } @@ -87,14 +79,14 @@ func TestScanLibrary_UnmatchedReasonAuthorNotInLibrary(t *testing.T) { // pointed at the case it was written for (#875): the author is in the library // but has no book the scan can attach a file to. func TestScanLibrary_UnmatchedReasonNoCandidateBooks(t *testing.T) { - s, _, _, authors, settings, libraryDir, ctx := unmatchedFixture(t) + s, _, _, authors, _, libraryDir, ctx := unmatchedFixture(t) author := &models.Author{ForeignID: "ol:weir", Name: "Andy Weir", SortName: "Weir, Andy", Monitored: true, MetadataProvider: "openlibrary"} if err := authors.Create(ctx, author); err != nil { t.Fatal(err) } - got := scanOneUnmatched(t, s, settings, ctx, libraryDir, "Andy Weir", "Project Hail Mary", "Project Hail Mary.epub") + got := scanOneUnmatched(t, s, ctx, libraryDir, "Andy Weir", "Project Hail Mary", "Project Hail Mary.epub") if got.Reason != unmatchedReasonNoCandidateBooks { t.Errorf("reason = %q, want %q", got.Reason, unmatchedReasonNoCandidateBooks) } @@ -103,7 +95,7 @@ func TestScanLibrary_UnmatchedReasonNoCandidateBooks(t *testing.T) { // TestScanLibrary_UnmatchedReasonNoTitleMatch covers the third case: the author // matched and has wanted books, but none of their titles is this file. func TestScanLibrary_UnmatchedReasonNoTitleMatch(t *testing.T) { - s, _, books, authors, settings, libraryDir, ctx := unmatchedFixture(t) + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) author := &models.Author{ForeignID: "ol:weir", Name: "Andy Weir", SortName: "Weir, Andy", Monitored: true, MetadataProvider: "openlibrary"} if err := authors.Create(ctx, author); err != nil { @@ -118,7 +110,7 @@ func TestScanLibrary_UnmatchedReasonNoTitleMatch(t *testing.T) { t.Fatal(err) } - got := scanOneUnmatched(t, s, settings, ctx, libraryDir, "Andy Weir", "The Martian", "The Martian.epub") + got := scanOneUnmatched(t, s, ctx, libraryDir, "Andy Weir", "The Martian", "The Martian.epub") if got.Reason != unmatchedReasonNoTitleMatch { t.Errorf("reason = %q, want %q", got.Reason, unmatchedReasonNoTitleMatch) } @@ -130,7 +122,7 @@ func TestScanLibrary_UnmatchedReasonNoTitleMatch(t *testing.T) { // about a file that had no title to match — the fix is to rename the file, and // the reason has to say that. func TestScanLibrary_UnmatchedReasonNoTitleParsed(t *testing.T) { - s, _, books, authors, settings, libraryDir, ctx := unmatchedFixture(t) + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) author := &models.Author{ForeignID: "ol:weir", Name: "Andy Weir", SortName: "Weir, Andy", Monitored: true, MetadataProvider: "openlibrary"} if err := authors.Create(ctx, author); err != nil { @@ -147,7 +139,7 @@ func TestScanLibrary_UnmatchedReasonNoTitleParsed(t *testing.T) { // A name that is nothing but a stripped parenthetical year parses to an // empty title, which is exactly the shape a release dump leaves behind. - got := scanOneUnmatched(t, s, settings, ctx, libraryDir, "Andy Weir", "(2021).epub") + got := scanOneUnmatched(t, s, ctx, libraryDir, "Andy Weir", "(2021).epub") if got.Reason != unmatchedReasonNoTitleParsed { t.Errorf("reason = %q, want %q", got.Reason, unmatchedReasonNoTitleParsed) } @@ -159,7 +151,7 @@ func TestScanLibrary_UnmatchedReasonNoTitleParsed(t *testing.T) { // match. The reason has to be computed from the same resolution the matcher // used, so this file is reported as a title miss. func TestScanLibrary_UnmatchedReasonUsesTheAuthorFallback(t *testing.T) { - s, _, books, authors, settings, libraryDir, ctx := unmatchedFixture(t) + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) author := &models.Author{ForeignID: "ol:enrigue", Name: "Álvaro Enrigue", SortName: "Enrigue, Álvaro", Monitored: true, MetadataProvider: "openlibrary"} if err := authors.Create(ctx, author); err != nil { @@ -185,7 +177,7 @@ func TestScanLibrary_UnmatchedReasonUsesTheAuthorFallback(t *testing.T) { s.ScanLibrary(ctx) - files := readUnmatchedFiles(t, ctx, settings) + files := readUnmatchedFiles(t, ctx, s) if len(files) != 1 { t.Fatalf("expected exactly one unmatched file, got %d (%+v)", len(files), files) } diff --git a/internal/importer/scanner_unmatched_test.go b/internal/importer/scanner_unmatched_test.go index 8069f690b..609ea8989 100644 --- a/internal/importer/scanner_unmatched_test.go +++ b/internal/importer/scanner_unmatched_test.go @@ -62,6 +62,7 @@ func unmatchedFixture(t *testing.T) (s *Scanner, downloads *db.DownloadRepo, boo libraryDir = t.TempDir() s = NewScanner(downloads, db.NewDownloadClientRepo(database), books, authors, db.NewHistoryRepo(database), libraryDir, "", "", "", "") s.WithSettings(settings) + s.WithUnmatchedUnits(db.NewUnmatchedUnitRepo(database)) if err := settings.Set(ctx, "import.mode", "copy"); err != nil { t.Fatal(err) } diff --git a/internal/importer/unit_boundaries.go b/internal/importer/unit_boundaries.go new file mode 100644 index 000000000..634967e2a --- /dev/null +++ b/internal/importer/unit_boundaries.go @@ -0,0 +1,85 @@ +package importer + +import ( + "os" + "path/filepath" + "regexp" + "strings" +) + +// The helpers below decide where one book ends and the next begins. Both the +// folder import scan (internal/api/scan_walk.go) and the library scan's +// unmatched grouping (unmatched_units.go) use them, so they live here once. + +// discFolderRe matches the disc/part subfolder names ("CD1", "Disc 2", +// "Part 03", "Vol. 1", or a bare "1"/"02") that split ONE audiobook across +// several directories. When every subdirectory of a folder looks like a disc, +// the folder is a single multi-disc audiobook rather than a shelf of separate +// books. +var discFolderRe = regexp.MustCompile(`(?i)^(cd|dis[ck]|part|pt|vol|volume|book|chapter|ch)\s*[._-]?\s*\d+$|^\d{1,2}$`) + +// IsDiscFolderName reports whether a directory's base name looks like one disc +// or part of a multi-disc audiobook. The name alone; see AllDiscFolders for +// the check that also requires audio beneath it. +func IsDiscFolderName(name string) bool { + return discFolderRe.MatchString(name) +} + +// AllDiscFolders reports whether every directory in dirs is a disc/part folder +// (by name) that actually holds audio somewhere beneath it. Both conditions are +// required so a shelf of audiobook folders with numeric-ish names isn't collapsed +// into one book, and an empty "CD1" placeholder doesn't fake a multi-disc set. +func AllDiscFolders(dirs []string) bool { + for _, d := range dirs { + if !discFolderRe.MatchString(filepath.Base(d)) { + return false + } + if !dirSubtreeHasAudio(d) { + return false + } + } + return len(dirs) > 0 +} + +// dirSubtreeHasAudio reports whether dir contains at least one audio file within +// a bounded number of entries, so a deep tree can't stall the disc-folder check. +func dirSubtreeHasAudio(dir string) bool { + const limit = 2000 + count := 0 + found := false + _ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + count++ + if count > limit { + return filepath.SkipAll + } + if !d.IsDir() && IsAudioFile(p) { + found = true + return filepath.SkipAll + } + return nil + }) + return found +} + +// SameStem reports whether every file shares one normalised base name (the file +// name without its extension, lower-cased). Title.epub and Title.mobi share the +// stem "title"; Dune.epub and Foundation.epub do not. +func SameStem(files []string) bool { + stem := func(p string) string { + b := filepath.Base(p) + return strings.ToLower(strings.TrimSuffix(b, filepath.Ext(b))) + } + if len(files) == 0 { + return false + } + first := stem(files[0]) + for _, f := range files[1:] { + if stem(f) != first { + return false + } + } + return true +} diff --git a/internal/importer/unmatched_discs_test.go b/internal/importer/unmatched_discs_test.go new file mode 100644 index 000000000..55f07c57e --- /dev/null +++ b/internal/importer/unmatched_discs_test.go @@ -0,0 +1,126 @@ +package importer + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/vavallee/bindery/internal/models" +) + +func touchAudio(t *testing.T, root string, rels ...string) []unmatchedScanFile { + t.Helper() + var out []unmatchedScanFile + for _, rel := range rels { + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + out = append(out, unmatchedScanFile{path: p, format: models.MediaTypeAudiobook, size: 1, mode: 0o644, title: filepath.Base(filepath.Dir(p))}) + } + return out +} + +// layoutEntry is one file or folder to create under a test root: a path +// ending in "/" is an empty folder, an .mp3 is an audio track, anything else +// a plain file. +func buildLayout(t *testing.T, root string, entries []string) []unmatchedScanFile { + t.Helper() + var audio []string + for _, e := range entries { + p := filepath.Join(root, e) + if strings.HasSuffix(e, "/") { + if err := os.MkdirAll(p, 0o755); err != nil { + t.Fatal(err) + } + continue + } + if filepath.Ext(e) == ".mp3" { + audio = append(audio, e) + continue + } + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + return touchAudio(t, root, audio...) +} + +// TestGroupUnmatched_DiscSetsFollowTheWalkerRule: a folder is one multi disc +// book only when every subfolder that holds audio is a CD, Disc or Disk +// folder. Wherever the call is unclear the books stay apart, because a set +// split in two can be adopted into one book, while separate books merged into +// one row cannot be adopted apart. +func TestGroupUnmatched_DiscSetsFollowTheWalkerRule(t *testing.T) { + tests := []struct { + name string + entries []string + units int + }{ + {"Author/Book/CD1,CD2", []string{"Andy Weir/Artemis/CD1/01.mp3", "Andy Weir/Artemis/CD2/01.mp3"}, 1}, + {"Author/Book/Disc 01,02", []string{"Andy Weir/Artemis/Disc 01/01.mp3", "Andy Weir/Artemis/Disc 02/01.mp3"}, 1}, + {"Author/Book/Disk1,Disk2", []string{"Andy Weir/Artemis/Disk1/01.mp3", "Andy Weir/Artemis/Disk2/01.mp3"}, 1}, + {"Author/Series/Book/CD1,CD2", []string{"Brandon Sanderson/Mistborn/Artemis/CD1/01.mp3", "Brandon Sanderson/Mistborn/Artemis/CD2/01.mp3"}, 1}, + {"Author/Series/1,2", []string{"Brandon Sanderson/Mistborn/1/01.mp3", "Brandon Sanderson/Mistborn/2/01.mp3"}, 2}, + {"Root/1,2", []string{"1/01.mp3", "2/01.mp3"}, 2}, + {"Root/Book/CD1,CD2", []string{"Artemis/CD1/01.mp3", "Artemis/CD2/01.mp3"}, 1}, + {"Author/Book/CD1,CD2 + Artwork with only a jpg", []string{"Andy Weir/Artemis/CD1/01.mp3", "Andy Weir/Artemis/CD2/01.mp3", "Andy Weir/Artemis/Artwork/cover.jpg"}, 1}, + {"Author/Book/CD1,CD2 + empty hidden folder", []string{"Andy Weir/Artemis/CD1/01.mp3", "Andy Weir/Artemis/CD2/01.mp3", "Andy Weir/Artemis/.hidden/"}, 1}, + {"Author/Book/CD1,CD2 + Synology @eaDir", []string{"Andy Weir/Artemis/CD1/01.mp3", "Andy Weir/Artemis/CD2/01.mp3", "Andy Weir/Artemis/@eaDir/SYNOINDEX_MEDIA_INFO"}, 1}, + {"Author/Book/Part 1,2", []string{"Andy Weir/Artemis/Part 1/01.mp3", "Andy Weir/Artemis/Part 2/01.mp3"}, 2}, + {"series of numbered books", []string{"Brandon Sanderson/Mistborn/Book 1/01.mp3", "Brandon Sanderson/Mistborn/Book 2/01.mp3"}, 2}, + {"a subfolder with audio that is not a disc", []string{"Andy Weir/Artemis/CD1/01.mp3", "Andy Weir/Artemis/Extras/01.mp3"}, 2}, + {"Root/CD1,CD2 never merges the root", []string{"CD1/01.mp3", "CD2/01.mp3"}, 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + groups, _ := groupUnmatched(buildLayout(t, root, tt.entries), []string{root}) + if len(groups) != tt.units { + var paths []string + for _, g := range groups { + paths = append(paths, g.unit.RelPath) + } + t.Fatalf("units = %d %v, want %d", len(groups), paths, tt.units) + } + if tt.units == 1 && (groups[0].unit.ParsedTitle != "Artemis" || groups[0].unit.UnitKind != "folder") { + t.Errorf("disc set unit = %+v, want a folder unit named after its book folder", groups[0].unit) + } + }) + } +} + +// TestScanLibrary_TrackedFolderDoesNotHideALaterSibling: a registered audio +// folder must not make a numbered folder beneath it count as already tracked; +// a Book 4 added later is a new book for the scan to consider. +func TestScanLibrary_TrackedFolderDoesNotHideALaterSibling(t *testing.T) { + s, _, books, authors, settings, libraryDir, ctx := unmatchedFixture(t) + author := &models.Author{ForeignID: "ol:bs", Name: "Brandon Sanderson", SortName: "Sanderson, Brandon", MetadataProvider: "openlibrary"} + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + owner := &models.Book{ForeignID: "ol:mb", AuthorID: author.ID, Title: "Mistborn", Status: models.BookStatusWanted, + MediaType: models.MediaTypeAudiobook, MetadataProvider: "openlibrary"} + if err := books.Create(ctx, owner); err != nil { + t.Fatal(err) + } + series := filepath.Join(libraryDir, "Brandon Sanderson", "Mistborn") + touchAudio(t, libraryDir, "Brandon Sanderson/Mistborn/01.mp3", "Brandon Sanderson/Mistborn/Book 4/01.mp3") + if err := books.AddBookFile(ctx, owner.ID, models.MediaTypeAudiobook, series); err != nil { + t.Fatal(err) + } + + s.ScanLibrary(ctx) + + b := readUnmatchedBlob(t, ctx, settings) + if b.FilesFound != 2 || b.AlreadyTracked != 1 { + t.Fatalf("counters = %+v, want the folder's own track tracked and Book 4 considered", b) + } +} diff --git a/internal/importer/unmatched_guards_test.go b/internal/importer/unmatched_guards_test.go new file mode 100644 index 000000000..37cb566d4 --- /dev/null +++ b/internal/importer/unmatched_guards_test.go @@ -0,0 +1,84 @@ +package importer + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/vavallee/bindery/internal/db" +) + +func unitRepoOf(t *testing.T, s *Scanner) *db.UnmatchedUnitRepo { + t.Helper() + repo, ok := s.unmatchedUnits.(*db.UnmatchedUnitRepo) + if !ok { + t.Fatal("scanner has no unmatched unit repo wired") + } + return repo +} + +func seedPendingUnit(t *testing.T, ctx context.Context, repo *db.UnmatchedUnitRepo, path string) { + t.Helper() + if _, err := repo.ReconcileScan(ctx, []db.UnmatchedUnitScan{{ + UnitPath: path, UnitKind: db.UnmatchedKindFile, Format: "ebook", FileCount: 1, MemberPaths: []string{path}, + }}, db.ReconcileScanOptions{SkipDeletion: true}); err != nil { + t.Fatal(err) + } +} + +func pendingPaths(t *testing.T, ctx context.Context, repo *db.UnmatchedUnitRepo) map[string]bool { + t.Helper() + units, _, err := repo.List(ctx, db.UnmatchedListQuery{Limit: 250}) + if err != nil { + t.Fatal(err) + } + out := make(map[string]bool, len(units)) + for _, u := range units { + out[u.UnitPath] = true + } + return out +} + +// TestScanLibrary_ZeroFileScanKeepsUnits: a scan that finds no files at all, +// the unmounted volume case, leaves every stored unit alone. +func TestScanLibrary_ZeroFileScanKeepsUnits(t *testing.T) { + s, _, _, _, _, libraryDir, ctx := unmatchedFixture(t) + repo := unitRepoOf(t, s) + kept := filepath.Join(libraryDir, "Gone", "Book.epub") + seedPendingUnit(t, ctx, repo, kept) + + s.ScanLibrary(ctx) + + if !pendingPaths(t, ctx, repo)[kept] { + t.Fatal("a scan that found no files removed a stored unit") + } +} + +// TestScanLibrary_TruncatedScanRemovesNothing: a scan over the unit cap did +// not see every unit, so a unit it did not list stays. +func TestScanLibrary_TruncatedScanRemovesNothing(t *testing.T) { + prev := maxUnmatchedUnits + maxUnmatchedUnits = 1 + t.Cleanup(func() { maxUnmatchedUnits = prev }) + + s, _, _, _, _, libraryDir, ctx := unmatchedFixture(t) + repo := unitRepoOf(t, s) + beyond := filepath.Join(libraryDir, "Zeta", "Unlisted.epub") + seedPendingUnit(t, ctx, repo, beyond) + for _, rel := range []string{"Alpha/One.txt", "Beta/Two.txt"} { + p := filepath.Join(libraryDir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + s.ScanLibrary(ctx) + + if !pendingPaths(t, ctx, repo)[beyond] { + t.Fatal("a truncated scan removed a unit it did not list") + } +} diff --git a/internal/importer/unmatched_units.go b/internal/importer/unmatched_units.go new file mode 100644 index 000000000..510b2e3d4 --- /dev/null +++ b/internal/importer/unmatched_units.go @@ -0,0 +1,438 @@ +package importer + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "regexp" + "slices" + "sort" + "strings" + "time" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/models" + "github.com/vavallee/bindery/internal/textutil" +) + +// Library adoption's side of the scan: the files the reconcile loop could not +// match are grouped into books and stored as unmatched_units rows, where an +// admin can adopt, ignore or leave them (see internal/api/adoption.go). + +// Scan bounds. Variables so a test can reach them without 50,000 files. +var ( + // maxUnmatchedFiles bounds how many unmatched files one scan groups. The + // old blob list stopped at 1000 files, which a single large audiobook + // could fill on its own (#2547). + maxUnmatchedFiles = 50000 + // maxUnmatchedUnits bounds how many rows one scan stores. + maxUnmatchedUnits = 20000 +) + +const ( + // candidateThreshold is the lowest title similarity offered as a + // suggestion. The reconcile itself needs 0.85; below that a person decides. + candidateThreshold = 0.60 + // maxCandidates is how many suggestions a unit keeps. + maxCandidates = 3 +) + +// UnmatchedUnitStore is where the scan records its unmatched units. +type UnmatchedUnitStore interface { + ReconcileScan(ctx context.Context, units []db.UnmatchedUnitScan, opts db.ReconcileScanOptions) (db.ReconcileScanResult, error) + Summary(ctx context.Context) (db.UnmatchedSummary, error) +} + +// WithUnmatchedUnits attaches the store the library scan writes unmatched +// units to. Nil leaves the scan counting unmatched files and storing nothing. +func (s *Scanner) WithUnmatchedUnits(store UnmatchedUnitStore) *Scanner { + s.unmatchedUnits = store + return s +} + +// ScanRunning reports whether a library scan is in flight, for the adoption +// page's scan status. +func (s *Scanner) ScanRunning() bool { + return s.scanRunning.Load() +} + +// walkedFile is what the library walk already knew about a file (P2): its +// size and mode come from the os.FileInfo filepath.Walk hands over, so +// grouping costs no extra stat. +type walkedFile struct { + size int64 + mode os.FileMode +} + +// unmatchedScanFile is one file the reconcile loop left unmatched, with what +// the loop parsed out of it. +type unmatchedScanFile struct { + path string + format string + size int64 + mode os.FileMode + title string + author string + layoutAuthor string + reason string +} + +// unmatchedCollector gathers unmatched files up to maxUnmatchedFiles. +type unmatchedCollector struct { + files []unmatchedScanFile + truncated bool +} + +func (c *unmatchedCollector) add(f unmatchedScanFile) { + if len(c.files) >= maxUnmatchedFiles { + c.truncated = true + return + } + c.files = append(c.files, f) +} + +// unmatchedGroup is one unit before candidates are ranked: the stored shape +// plus the member whose parse speaks for the unit. +type unmatchedGroup struct { + unit db.UnmatchedUnitScan + rep unmatchedScanFile +} + +// scanRootFor returns the longest root that contains path, or "". +func scanRootFor(path string, roots []string) string { + best := "" + for _, r := range roots { + if r != "" && pathUnderDir(path, r) && len(r) > len(best) { + best = r + } + } + return best +} + +// discSetNameRe is the disc folder names library adoption treats as parts of +// one book: CD 1, Disc 2, Disk 3. It is narrower than IsDiscFolderName, which +// also accepts Book, Part, Vol, Chapter and bare numbers: in a library those +// name separate books of a series as often as discs ("Mistborn/Book 1", +// "Series/1"). Wherever the call is unclear adoption keeps folders apart, +// because a split set can be adopted piece by piece into one book, while +// separate books merged into one row cannot be adopted apart. +var discSetNameRe = regexp.MustCompile(`(?i)^(cd|dis[ck])\s*[._-]?\s*\d+$`) + +// ignoredSubfolder reports whether a subfolder has no say in whether its +// parent is a disc set: hidden and system folders (".hidden", Synology +// "@eaDir", QNAP ".@__thumb", "#recycle") and folders with no audio anywhere +// beneath them (an Artwork folder of jpgs). +func ignoredSubfolder(dir string) bool { + name := filepath.Base(dir) + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "@") || strings.HasPrefix(name, "#") { + return true + } + return !dirSubtreeHasAudio(dir) +} + +// discSetChecker answers, once per folder, whether a folder is one multi disc +// audiobook: every subfolder that counts (ignoredSubfolder) has a disc name +// and holds audio, as the folder import walker's AllDiscFolders requires. A +// library root is never a disc set; a book folder directly under one can be. +func discSetChecker(roots []string) func(folder string) bool { + cache := make(map[string]bool) + return func(folder string) bool { + if v, ok := cache[folder]; ok { + return v + } + result := false + isRoot := false + for _, r := range roots { + if r != "" && filepath.Clean(r) == folder { + isRoot = true + } + } + if !isRoot && scanRootFor(folder, roots) != "" { + if entries, err := os.ReadDir(folder); err == nil { + var discs []string + names := true + for _, e := range entries { + if !e.IsDir() { + continue + } + dir := filepath.Join(folder, e.Name()) + if ignoredSubfolder(dir) { + continue + } + discs = append(discs, dir) + names = names && discSetNameRe.MatchString(e.Name()) + } + result = names && AllDiscFolders(discs) + } + } + cache[folder] = result + return result + } +} + +// unitKeyFor decides which unit a file belongs to (#2547). The unit is a book, +// not a file: +// +// - audio groups by its folder, so 193 tracks are one row; +// - audio in a disc folder groups by the folder above it, when that folder +// is a disc set (discSetChecker); +// - loose audio directly in a library root has no book folder, so each file +// stands alone, as in the folder import scan; +// - ebooks group by folder and file stem, so Title.epub and Title.mobi are +// one book in two formats. +// +// It returns the grouping key, and for a folder unit the folder path. +func unitKeyFor(f unmatchedScanFile, root string, isDiscSet func(string) bool) (key, folder string) { + parent := filepath.Dir(f.path) + if f.format == models.MediaTypeAudiobook { + if parent == root { + return "file\x00" + f.path, "" + } + if discSetNameRe.MatchString(filepath.Base(parent)) && isDiscSet(filepath.Dir(parent)) { + parent = filepath.Dir(parent) + } + return "folder\x00" + parent, parent + } + base := filepath.Base(f.path) + return "stem\x00" + parent + "\x00" + strings.ToLower(strings.TrimSuffix(base, filepath.Ext(base))), "" +} + +// groupUnmatched turns unmatched files into book units. Every file lands in +// exactly one unit, and the units' file counts sum to the number of files +// given (TestGroupUnmatched_Property). Units come back ordered by path. More +// than maxUnmatchedUnits units is truncated to that many. +func groupUnmatched(files []unmatchedScanFile, roots []string) (groups []unmatchedGroup, truncated bool) { + type acc struct { + folder string + root string + members []unmatchedScanFile + } + byKey := make(map[string]*acc) + var keys []string + isDiscSet := discSetChecker(roots) + for _, f := range files { + root := scanRootFor(f.path, roots) + key, folder := unitKeyFor(f, root, isDiscSet) + a := byKey[key] + if a == nil { + a = &acc{folder: folder, root: root} + byKey[key] = a + keys = append(keys, key) + } + a.members = append(a.members, f) + } + + groups = make([]unmatchedGroup, 0, len(keys)) + for _, key := range keys { + a := byKey[key] + slices.SortFunc(a.members, func(x, y unmatchedScanFile) int { return strings.Compare(x.path, y.path) }) + u := db.UnmatchedUnitScan{ + UnitPath: a.members[0].path, + UnitKind: db.UnmatchedKindFile, + Format: a.members[0].format, + FileCount: len(a.members), + RootPath: a.root, + } + if a.folder != "" { + u.UnitPath, u.UnitKind = a.folder, db.UnmatchedKindFolder + } + u.MemberPaths = make([]string, len(a.members)) + for i, m := range a.members { + u.MemberPaths[i] = m.path + u.SizeBytes += m.size + } + if a.root != "" { + if rel, err := filepath.Rel(a.root, u.UnitPath); err == nil { + u.RelPath = filepath.ToSlash(rel) + } + if rel, err := filepath.Rel(a.root, a.members[0].path); err == nil { + if parts := strings.Split(filepath.ToSlash(rel), "/"); len(parts) >= 2 { + u.AuthorFolder = parts[0] + } + } + } + rep := representative(a.members) + // A disc set's tracks parse their title from the disc folder ("CD 1"); + // the book is the folder above it, so the unit is named after that. + if a.folder != "" && filepath.Dir(a.members[0].path) != a.folder { + rep.title = cleanLayoutTitle(filepath.Base(a.folder)) + } + u.ParsedTitle, u.ParsedAuthor, u.Reason = rep.title, rep.author, rep.reason + groups = append(groups, unmatchedGroup{unit: u, rep: rep}) + } + sort.Slice(groups, func(i, j int) bool { return groups[i].unit.UnitPath < groups[j].unit.UnitPath }) + if len(groups) > maxUnmatchedUnits { + groups, truncated = groups[:maxUnmatchedUnits], true + } + return groups, truncated +} + +// representative picks the member whose parse speaks for the unit: the one +// carrying the title most members agree on. For an audiobook folder that is +// the folder's title rather than one track's chapter name. Members arrive +// sorted, so ties go to the first path. +func representative(members []unmatchedScanFile) unmatchedScanFile { + counts := make(map[string]int, len(members)) + best := members[0] + for _, m := range members { + counts[m.title]++ + if counts[m.title] > counts[best.title] { + best = m + } + } + return best +} + +// eligibleUnmatched keeps only the files that may become rows (S10): regular +// files, by the mode the walk read with Lstat, whose resolved folder lies in +// a scanned root. A symlink inside the library would otherwise become a +// tracked book file that the download and OPDS routes serve from wherever it +// points. Each folder is resolved once. +func eligibleUnmatched(files []unmatchedScanFile, roots []string) []unmatchedScanFile { + realRoots := make([]string, 0, len(roots)) + for _, r := range roots { + if r == "" { + continue + } + if rr, err := filepath.EvalSymlinks(r); err == nil { + realRoots = append(realRoots, rr) + } + } + resolved := make(map[string]bool) + out := files[:0:0] + for _, f := range files { + if !f.mode.IsRegular() { + slog.Debug("library scan: unmatched entry is not a regular file; not listing it", "path", f.path) + continue + } + dir := filepath.Dir(f.path) + ok, seen := resolved[dir] + if !seen { + if real, err := filepath.EvalSymlinks(dir); err == nil { + for _, r := range realRoots { + if pathUnderDir(real, r) { + ok = true + break + } + } + } + resolved[dir] = ok + } + if !ok { + slog.Debug("library scan: unmatched file resolves outside the library; not listing it", "path", f.path) + continue + } + out = append(out, f) + } + return out +} + +// rankCandidates returns up to maxCandidates catalogue books whose title +// scores at least candidateThreshold against normParsed, best first. It uses +// the reconcile's own measure (Jaro-Winkler over normalizeTitle) and its own +// candidate set: the books of the authors the file's author resolved to, or +// every reconcile candidate when the file named no author. In that last case +// titles of wildly different length are skipped, the same cheap gate the +// reconcile uses. No provider is asked anything. +func rankCandidates(normParsed string, wanted []scanBook, byAuthor map[int64][]int, authorSet map[int64]bool) []db.UnmatchedCandidate { + if normParsed == "" { + return nil + } + var out []db.UnmatchedCandidate + consider := func(sb *scanBook) { + score := textutil.JaroWinkler(sb.normTitle, normParsed) + if score < candidateThreshold { + return + } + out = append(out, db.UnmatchedCandidate{BookID: sb.book.ID, Score: score}) + } + if authorSet == nil { + for i := range wanted { + lo, hi := wanted[i].normLen, len(normParsed) + if lo > hi { + lo, hi = hi, lo + } + if lo == 0 || lo*4 < hi { + continue + } + consider(&wanted[i]) + } + } else { + ids := make([]int64, 0, len(authorSet)) + for id := range authorSet { + ids = append(ids, id) + } + slices.Sort(ids) + for _, id := range ids { + for _, idx := range byAuthor[id] { + consider(&wanted[idx]) + } + } + } + slices.SortStableFunc(out, func(a, b db.UnmatchedCandidate) int { + switch { + case a.Score > b.Score: + return -1 + case a.Score < b.Score: + return 1 + } + return 0 + }) + if len(out) > maxCandidates { + out = out[:maxCandidates] + } + return out +} + +// unitCounts is what the scan result blob reports about stored units. +type unitCounts struct { + pending int + ignored int + truncated bool +} + +// recordUnmatchedUnits groups a finished scan's unmatched files and stores +// them. candidatesFor ranks suggestions for one unit's representative parse. +func (s *Scanner) recordUnmatchedUnits(ctx context.Context, c *unmatchedCollector, roots, rootsWithFiles []string, startedAt time.Time, + candidatesFor func(title, author, layoutAuthor string) []db.UnmatchedCandidate) unitCounts { + if s.unmatchedUnits == nil { + return unitCounts{} + } + groups, unitsTruncated := groupUnmatched(eligibleUnmatched(c.files, roots), roots) + units := make([]db.UnmatchedUnitScan, len(groups)) + for i, g := range groups { + units[i] = g.unit + units[i].Candidates = candidatesFor(g.rep.title, g.rep.author, g.rep.layoutAuthor) + } + truncated := c.truncated || unitsTruncated + // A truncated scan did not see every unit, so it removes and purges + // nothing: a unit beyond the cap is unlisted, not gone. + res, err := s.unmatchedUnits.ReconcileScan(ctx, units, db.ReconcileScanOptions{ + StartedAt: startedAt, + SkipDeletion: truncated, + RootsWithFiles: rootsWithFiles, + ConfiguredRoots: roots, + }) + if err != nil { + slog.Warn("library scan: failed to store unmatched units", "error", err) + return s.unmatchedUnitCounts(ctx, truncated) + } + slog.Info("library scan: stored unmatched units", "units", res.Upserted, "removed", res.RemovedPending, + "purged", res.Purged, "pending", res.Pending, "ignored", res.Ignored, "truncated", truncated) + return unitCounts{pending: res.Pending, ignored: res.Ignored, truncated: truncated} +} + +// unmatchedUnitCounts reads the counts without changing any row, for a scan +// that found no files or failed: those must not delete anyone's decisions. +func (s *Scanner) unmatchedUnitCounts(ctx context.Context, truncated bool) unitCounts { + if s.unmatchedUnits == nil { + return unitCounts{} + } + sum, err := s.unmatchedUnits.Summary(ctx) + if err != nil { + slog.Warn("library scan: failed to count unmatched units", "error", err) + return unitCounts{truncated: truncated} + } + return unitCounts{pending: sum.Pending, ignored: sum.Ignored, truncated: truncated} +} diff --git a/internal/importer/unmatched_units_test.go b/internal/importer/unmatched_units_test.go new file mode 100644 index 000000000..24a7a345f --- /dev/null +++ b/internal/importer/unmatched_units_test.go @@ -0,0 +1,329 @@ +package importer + +import ( + "fmt" + "math/rand/v2" + "net/http" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/vavallee/bindery/internal/models" +) + +func ebookFile(path string) unmatchedScanFile { + return unmatchedScanFile{path: path, format: models.MediaTypeEbook, size: 100, mode: 0o644, title: filepath.Base(path)} +} + +func audioFile(path, title string) unmatchedScanFile { + return unmatchedScanFile{path: path, format: models.MediaTypeAudiobook, size: 1000, mode: 0o644, title: title} +} + +// TestGroupUnmatched_Table pins where one book ends and the next begins. +func TestGroupUnmatched_Table(t *testing.T) { + root := "/lib" + type want struct { + path, kind, format, folder, rel, title string + files int + } + tests := []struct { + name string + files []unmatchedScanFile + want []want + }{ + { + name: "audio tracks in a book folder are one folder unit", + files: []unmatchedScanFile{ + audioFile("/lib/Weir/Hail Mary/02.mp3", "Chapter 2"), + audioFile("/lib/Weir/Hail Mary/01.mp3", "Project Hail Mary"), + audioFile("/lib/Weir/Hail Mary/03.mp3", "Project Hail Mary"), + }, + want: []want{{"/lib/Weir/Hail Mary", "folder", "audiobook", "Weir", "Weir/Hail Mary", "Project Hail Mary", 3}}, + }, + { + name: "a disc folder straight under the root is its own unit", + files: []unmatchedScanFile{ + audioFile("/lib/CD1/01.mp3", "Something"), + audioFile("/lib/CD1/02.mp3", "Something"), + }, + want: []want{{"/lib/CD1", "folder", "audiobook", "CD1", "CD1", "Something", 2}}, + }, + { + name: "loose audio at the root stands alone", + files: []unmatchedScanFile{ + audioFile("/lib/a.m4b", "A"), + audioFile("/lib/b.m4b", "B"), + }, + want: []want{ + {"/lib/a.m4b", "file", "audiobook", "", "a.m4b", "A", 1}, + {"/lib/b.m4b", "file", "audiobook", "", "b.m4b", "B", 1}, + }, + }, + { + name: "same stem ebooks are one book, other stems are separate", + files: []unmatchedScanFile{ + ebookFile("/lib/Herbert/Dune.mobi"), + ebookFile("/lib/Herbert/Dune.epub"), + ebookFile("/lib/Herbert/Children of Dune.epub"), + }, + want: []want{ + {"/lib/Herbert/Children of Dune.epub", "file", "ebook", "Herbert", "Herbert/Children of Dune.epub", "Children of Dune.epub", 1}, + {"/lib/Herbert/Dune.epub", "file", "ebook", "Herbert", "Herbert/Dune.epub", "Dune.epub", 2}, + }, + }, + { + name: "an ebook beside audio is a separate book unit", + files: []unmatchedScanFile{ + audioFile("/lib/X/Book/01.mp3", "Book"), + ebookFile("/lib/X/Book/Book.epub"), + }, + want: []want{ + {"/lib/X/Book", "folder", "audiobook", "X", "X/Book", "Book", 1}, + {"/lib/X/Book/Book.epub", "file", "ebook", "X", "X/Book/Book.epub", "Book.epub", 1}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + groups, truncated := groupUnmatched(tt.files, []string{root}) + if truncated { + t.Fatal("unexpected truncation") + } + if len(groups) != len(tt.want) { + t.Fatalf("got %d units, want %d: %+v", len(groups), len(tt.want), groups) + } + for i, w := range tt.want { + u := groups[i].unit + got := want{u.UnitPath, u.UnitKind, u.Format, u.AuthorFolder, u.RelPath, u.ParsedTitle, u.FileCount} + if got != w { + t.Errorf("unit %d = %+v, want %+v", i, got, w) + } + if len(u.MemberPaths) != u.FileCount || !slices.IsSorted(u.MemberPaths) { + t.Errorf("unit %d members %v do not match its file count", i, u.MemberPaths) + } + } + }) + } +} + +// TestGroupUnmatched_Property is T2: for random trees, every file lands in +// exactly one unit and the units' file counts and sizes sum to the input. +func TestGroupUnmatched_Property(t *testing.T) { + rng := rand.New(rand.NewPCG(2547, 1436)) + dirs := []string{"/lib", "/lib/A", "/lib/A/B1", "/lib/A/B1/CD1", "/lib/A/B1/CD2", "/lib/A/B2", "/lib/C", "/lib/C/Part 3", "/audio", "/audio/D/E"} + stems := []string{"one", "two", "One", "three"} + exts := []string{".epub", ".mobi", ".mp3", ".m4b", ".pdf"} + for trial := range 300 { + n := rng.IntN(60) + seen := map[string]bool{} + var files []unmatchedScanFile + var wantSize int64 + for range n { + p := filepath.Join(dirs[rng.IntN(len(dirs))], fmt.Sprintf("%s%d%s", stems[rng.IntN(len(stems))], rng.IntN(3), exts[rng.IntN(len(exts))])) + if seen[p] { + continue + } + seen[p] = true + f := ebookFile(p) + f.title = stems[rng.IntN(len(stems))] + if detectDownloadFormat([]string{p}) == models.MediaTypeAudiobook { + f.format = models.MediaTypeAudiobook + } + f.size = int64(rng.IntN(5000)) + wantSize += f.size + files = append(files, f) + } + groups, _ := groupUnmatched(files, []string{"/lib", "/audio"}) + placed := map[string]int{} + total := 0 + var size int64 + for _, g := range groups { + total += g.unit.FileCount + size += g.unit.SizeBytes + for _, m := range g.unit.MemberPaths { + placed[m]++ + } + } + if total != len(files) || size != wantSize { + t.Fatalf("trial %d: units hold %d files / %d bytes, input %d / %d", trial, total, size, len(files), wantSize) + } + for _, f := range files { + if placed[f.path] != 1 { + t.Fatalf("trial %d: %s placed %d times", trial, f.path, placed[f.path]) + } + } + } +} + +// TestEligibleUnmatched_SymlinkOutsideRoot is S10: a symlink inside the +// library never becomes a row, and neither does a file whose folder resolves +// outside every root. +func TestEligibleUnmatched_SymlinkOutsideRoot(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + secret := filepath.Join(outside, "secret.epub") + if err := os.WriteFile(secret, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "Author"), 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "Author", "Linked.epub") + if err := os.Symlink(secret, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + linkedDir := filepath.Join(root, "Elsewhere") + if err := os.Symlink(outside, linkedDir); err != nil { + t.Fatal(err) + } + real := filepath.Join(root, "Author", "Real.epub") + if err := os.WriteFile(real, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + linkInfo, err := os.Lstat(link) + if err != nil { + t.Fatal(err) + } + + in := []unmatchedScanFile{ + {path: link, format: models.MediaTypeEbook, mode: linkInfo.Mode()}, + {path: filepath.Join(linkedDir, "secret.epub"), format: models.MediaTypeEbook, mode: 0o644}, + {path: real, format: models.MediaTypeEbook, mode: 0o644}, + } + got := eligibleUnmatched(in, []string{root}) + if len(got) != 1 || got[0].path != real { + t.Fatalf("eligible = %+v, want only the real file", got) + } +} + +// TestScanLibrary_SymlinkedBookIsNotListed runs S10 through a real scan. +func TestScanLibrary_SymlinkedBookIsNotListed(t *testing.T) { + s, _, _, _, _, libraryDir, ctx := unmatchedFixture(t) + outside := t.TempDir() + secret := filepath.Join(outside, "Stolen.epub") + writeEpubAt(t, secret, "", "", "") + if err := os.MkdirAll(filepath.Join(libraryDir, "Someone"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(secret, filepath.Join(libraryDir, "Someone", "Stolen.epub")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + writeEpubAt(t, filepath.Join(libraryDir, "Someone", "Honest.epub"), "", "", "") + + s.ScanLibrary(ctx) + + units := readUnmatchedFiles(t, ctx, s) + if len(units) != 1 || filepath.Base(units[0].UnitPath) != "Honest.epub" { + t.Fatalf("units = %+v, want only Honest.epub", units) + } +} + +// TestScanLibrary_RanksCandidatesFromTheCatalogue: a unit close to a book of +// its author, but below the reconcile's 0.85, is stored with that book as a +// suggestion. Only in memory catalogue rows are used. +func TestScanLibrary_RanksCandidatesFromTheCatalogue(t *testing.T) { + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) + author := &models.Author{ForeignID: "ol:weir", Name: "Andy Weir", SortName: "Weir, Andy", MetadataProvider: "openlibrary"} + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + var martian int64 + for _, title := range []string{"The Martian", "Artemis", "Project Hail Mary"} { + b := &models.Book{ForeignID: "ol:" + title, AuthorID: author.ID, Title: title, Status: models.BookStatusWanted, + MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := books.Create(ctx, b); err != nil { + t.Fatal(err) + } + if title == "The Martian" { + martian = b.ID + } + } + path := filepath.Join(libraryDir, "Andy Weir", "A Martyrs Tale", "A Martyrs Tale.epub") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + writeEpubAt(t, path, "", "", "") + + s.ScanLibrary(ctx) + + units := readUnmatchedFiles(t, ctx, s) + if len(units) != 1 { + t.Fatalf("units = %+v", units) + } + c := units[0].Candidates + if len(c) == 0 || c[0].BookID != martian || c[0].Score < candidateThreshold || c[0].Score >= 0.85 { + t.Fatalf("candidates = %+v, want The Martian first with a score in [0.60, 0.85)", c) + } + if units[0].AuthorFolder != "Andy Weir" || units[0].TopScore != c[0].Score { + t.Errorf("unit = %+v", units[0]) + } +} + +// countingTransport fails the test on any outbound request. +type countingTransport struct{ calls int } + +func (c *countingTransport) RoundTrip(*http.Request) (*http.Response, error) { + c.calls++ + return nil, fmt.Errorf("unexpected outbound request during a library scan") +} + +// TestScanLibrary_MakesNoOutboundCalls is T5 for the scan: grouping and +// candidate ranking never reach a metadata provider. +func TestScanLibrary_MakesNoOutboundCalls(t *testing.T) { + spy := &countingTransport{} + prev := http.DefaultTransport + http.DefaultTransport = spy + t.Cleanup(func() { http.DefaultTransport = prev }) + + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) + author := &models.Author{ForeignID: "ol:x", Name: "Ann Leckie", SortName: "Leckie, Ann", MetadataProvider: "openlibrary"} + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + if err := books.Create(ctx, &models.Book{ForeignID: "ol:aj", AuthorID: author.ID, Title: "Ancillary Justice", + Status: models.BookStatusWanted, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"}); err != nil { + t.Fatal(err) + } + for _, rel := range []string{"Ann Leckie/Provenance.epub", "Nobody/Unknown.epub"} { + p := filepath.Join(libraryDir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + writeEpubAt(t, p, "", "", "") + } + + s.ScanLibrary(ctx) + + if spy.calls != 0 { + t.Fatalf("outbound calls during scan = %d, want 0", spy.calls) + } + if n := len(readUnmatchedFiles(t, ctx, s)); n != 2 { + t.Fatalf("units = %d, want 2", n) + } +} + +// BenchmarkGroupUnmatched is T6: grouping at the 50,000 file cap, a library +// of audiobook folders and loose ebooks. +func BenchmarkGroupUnmatched(b *testing.B) { + files := make([]unmatchedScanFile, 0, maxUnmatchedFiles) + for i := 0; len(files) < maxUnmatchedFiles; i++ { + author := fmt.Sprintf("/lib/Author %03d", i%500) + if i%2 == 0 { + for track := range 20 { + files = append(files, audioFile(fmt.Sprintf("%s/Book %05d/%02d.mp3", author, i, track), fmt.Sprintf("Book %05d", i))) + } + } else { + files = append(files, ebookFile(fmt.Sprintf("%s/Book %05d.epub", author, i))) + } + } + files = files[:maxUnmatchedFiles] + roots := []string{"/lib"} + b.ResetTimer() + for b.Loop() { + groups, _ := groupUnmatched(files, roots) + if len(groups) == 0 { + b.Fatal("no groups") + } + } +} diff --git a/web/src/App.test.tsx b/web/src/App.test.tsx index c91551a5a..8a9841d15 100644 --- a/web/src/App.test.tsx +++ b/web/src/App.test.tsx @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { render, screen, fireEvent } from '@testing-library/react' import App from './App' +import { api } from './api/client' const { authState, logoutMock } = vi.hoisted(() => ({ authState: { @@ -50,6 +51,8 @@ vi.mock('./api/client', () => ({ // SetupBanner (mounted in the admin shell) probes these on mount. listIndexers: vi.fn().mockResolvedValue([]), listDownloadClients: vi.fn().mockResolvedValue([]), + // The Import nav badge reads this once for an admin. + unmatchedSummary: vi.fn().mockResolvedValue({ pending: 0, pendingFiles: 0, ignored: 0, adopted: 0, scan: {} }), }, })) @@ -282,3 +285,21 @@ describe('Shell — mobile navigation', () => { expect(btn.innerHTML).toContain('M4 6h16M4 12h16M4 18h16') }) }) + +describe('Shell: Import nav badge', () => { + // Admins see how many library books still need a decision on the Import + // entry. The count comes from an admin only route, so nobody else asks. + it('shows the unmatched count for an admin', async () => { + vi.mocked(api.unmatchedSummary).mockResolvedValue({ pending: 38, pendingFiles: 412, ignored: 0, adopted: 0, scan: {} as never }) + authState.value = { status: { authenticated: true, setupRequired: false, mode: 'enabled' }, logout: logoutMock, isAdmin: true } + renderShell() + const desktopNav = document.querySelector('nav.hidden.xl\\:flex') as HTMLElement + const importLink = () => Array.from(desktopNav.querySelectorAll('a')).find(a => a.getAttribute('href') === '/import') + await vi.waitFor(() => expect(importLink()?.textContent).toBe('Import38')) + }) + + it('never asks for the count as a non admin', () => { + renderShell() + expect(api.unmatchedSummary).not.toHaveBeenCalled() + }) +}) diff --git a/web/src/App.tsx b/web/src/App.tsx index 0c15da6ec..7bfc5c168 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -12,6 +12,7 @@ import Logo from './components/Logo' import SetupBanner from './components/SetupBanner' import VersionBadge from './components/VersionBadge' import WhatsNewToast from './components/WhatsNewToast' +import { useUnmatchedCount } from './components/useUnmatchedCount' import { useTheme } from './theme' // Route-scoped error boundary: a render crash in one page shows an inline error @@ -35,7 +36,7 @@ const BooksPage = lazy(() => import('./pages/BooksPage')) const BookDetailPage = lazy(() => import('./pages/BookDetailPage')) const WantedPage = lazy(() => import('./pages/WantedPage')) const QueuePage = lazy(() => import('./pages/QueuePage')) -const ManualImportPage = lazy(() => import('./pages/ManualImportPage')) +const ImportPage = lazy(() => import('./pages/import/ImportPage')) const SettingsPage = lazy(() => import('./pages/SettingsPage')) const UsersPage = lazy(() => import('./pages/UsersPage')) const HistoryPage = lazy(() => import('./pages/HistoryPage')) @@ -109,6 +110,22 @@ function Shell() { const [menuOpen, setMenuOpen] = useState(false) const { status, logout, isAdmin } = useAuth() const signedIn = !!status?.authenticated && status.mode !== 'disabled' + // Books the library scan could not match, on the Import nav entry (admins + // only; the count comes from an admin only route). + const unmatched = useUnmatchedCount(isAdmin) + const navLabel = (key: string) => ( + <> + {t(`nav.${key}`)} + {key === 'import' && unmatched > 0 && ( + + {unmatched > 999 ? '999+' : unmatched} + + )} + + ) useEffect(() => { api.status().then(s => { @@ -140,7 +157,7 @@ function Shell() { @@ -233,7 +250,7 @@ function Shell() { className={mobileLinkClass} onClick={() => setMenuOpen(false)} > - {t(`nav.${item.key}`)} + {navLabel(item.key)} ))} } /> } /> } /> - } /> + } /> } /> } /> } /> diff --git a/web/src/api/adoption.ts b/web/src/api/adoption.ts new file mode 100644 index 000000000..9a1aabafd --- /dev/null +++ b/web/src/api/adoption.ts @@ -0,0 +1,144 @@ +import { request } from './core' + +// Library adoption (#2547): the books a library scan could not match, grouped +// one row per book, and the decisions an admin makes about them. Every route +// is admin only. No request carries a path; rows are addressed by id. + +export type AdoptionState = 'pending' | 'adopted' | 'ignored' + +export type AdoptionReason = + | 'author_not_in_library' + | 'no_candidate_books' + | 'no_title_match' + | 'no_title_parsed' + +export interface AdoptionBookRef { + id: number + title: string + authorId: number + authorName: string + imageUrl?: string + status: string + mediaType: string + monitored: boolean +} + +export interface AdoptionCandidate { + book: AdoptionBookRef + score: number +} + +export interface AdoptionItem { + id: number + kind: 'file' | 'folder' + format: 'ebook' | 'audiobook' + fileCount: number + sizeBytes: number + relPath: string + rootPath: string + authorFolder: string + parsedTitle: string + parsedAuthor: string + reason: AdoptionReason | '' + candidates: AdoptionCandidate[] + topScore: number + state: AdoptionState | 'adopting' | 'undoing' + book?: AdoptionBookRef + bookCreated: boolean + authorCreated: boolean + members: string[] + firstSeenAt: string + resolvedAt?: string + // Set when an outcome needs explaining, such as Undo keeping a book that + // has been used since the adoption added it. + message?: string +} + +export interface AdoptionFacetCount { + value: string + count: number +} + +export interface AdoptionFolderFacet { + folder: string + units: number + files: number + notInLibrary: number + author: string +} + +export interface AdoptionFacets { + reasons: AdoptionFacetCount[] + formats: AdoptionFacetCount[] + folders: AdoptionFolderFacet[] +} + +export interface AdoptionSummary { + pending: number + pendingFiles: number + ignored: number + adopted: number +} + +export interface AdoptionScanStatus { + ran: boolean + ranAt?: string + running: boolean + filesFound: number + truncated: boolean + error?: string + noFilesFound: boolean +} + +export interface AdoptionListResponse { + items: AdoptionItem[] + total: number + facets?: AdoptionFacets + summary: AdoptionSummary + scan: AdoptionScanStatus +} + +export type AdoptionSort = 'score' | 'title' | 'folder' | 'files' | 'size' | 'seen' + +export interface AdoptionListParams { + state?: AdoptionState + reason?: string + authorFolder?: string + format?: string + search?: string + sort?: AdoptionSort + dir?: 'asc' | 'desc' + limit?: number + offset?: number + facets?: boolean +} + +// What a row is adopted as: a book already in the library, or a metadata +// result the server adds (unmonitored, in the adopted format) first. +export type AdoptTarget = + | { bookId: number; format?: string } + | { foreignBookId: string; foreignAuthorId: string; authorName: string; format?: string } + +export const adoptionApi = { + listUnmatched: (p: AdoptionListParams = {}) => { + const q = new URLSearchParams() + for (const [k, v] of Object.entries(p)) { + if (v === undefined || v === '' || v === false) continue + q.set(k, v === true ? '1' : String(v)) + } + const qs = q.toString() + return request(`/library/unmatched${qs ? `?${qs}` : ''}`) + }, + unmatchedSummary: () => + request('/library/unmatched/summary'), + adoptUnit: (id: number, target: AdoptTarget) => + request(`/library/unmatched/${id}/adopt`, { method: 'POST', body: JSON.stringify(target) }), + undoAdoption: (id: number) => + request(`/library/unmatched/${id}/undo`, { method: 'POST' }), + ignoreUnit: (id: number) => + request(`/library/unmatched/${id}/ignore`, { method: 'POST' }), + unignoreUnit: (id: number) => + request(`/library/unmatched/${id}/unignore`, { method: 'POST' }), + ignoreFolder: (authorFolder: string) => + request<{ ignored: number }>('/library/unmatched/ignore', { method: 'POST', body: JSON.stringify({ authorFolder }) }), +} diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 4413db57f..e468bac55 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -30,6 +30,7 @@ import { grimmoryApi } from './grimmory' import { absApi } from './abs' import { importListsApi } from './importlists' import { recommendationsApi } from './recommendations' +import { adoptionApi } from './adoption' // Shared core: public constant, error class, helpers, and CSRF init. export { ApiError, BINDERY_BASE, isNoDownloadClientError, initCSRF } from './core' @@ -59,6 +60,7 @@ export type * from './grimmory' export type * from './abs' export type * from './importlists' export type * from './recommendations' +export type * from './adoption' // The single `api` object, composed from each domain's slice. Key order mirrors // the original file's section ordering; all method names are unique across @@ -88,4 +90,5 @@ export const api = { ...absApi, ...importListsApi, ...recommendationsApi, + ...adoptionApi, } diff --git a/web/src/api/library.ts b/web/src/api/library.ts index ad2507c1d..619c1aaac 100644 --- a/web/src/api/library.ts +++ b/web/src/api/library.ts @@ -42,10 +42,12 @@ export const libraryApi = { reconciled: number unmatched: number tag_read_failed?: number - // reason is the scanner's per-file diagnostic (#1958): 'author_not_in_library', - // 'no_candidate_books', 'no_title_match' or 'no_title_parsed'. Optional — - // older cached scan results were written before it existed. - unmatched_files?: Array<{ path: string; parsed_title: string; parsed_author: string; reason?: string }> + // Books the scan could not match (library adoption, /library/unmatched) + // and how many of them are ignored. unmatched_files is always empty now + // and kept only so a cached older bundle still parses. + unmatched_units?: number + ignored_units?: number + units_truncated?: boolean // Additive (feat/library-scan-visibility): the resolved roots the scan // walked and an explicit zero-files signal. Optional so older cached scan // results (persisted before these fields existed) still parse. diff --git a/web/src/components/ImportHints.test.tsx b/web/src/components/ImportHints.test.tsx new file mode 100644 index 000000000..118c3e069 --- /dev/null +++ b/web/src/components/ImportHints.test.tsx @@ -0,0 +1,20 @@ +import { render, screen } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' +import { MemoryRouter } from 'react-router' +import ImportHints from './ImportHints' + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ + t: (key: string) => ({ 'importHints.manualImport': 'Import from a folder', 'importHints.scanLibrary': 'Review my library' }[key] ?? key), + }), +})) + +// The empty Queue and Wanted pages point new users at the Import page, which +// now holds both ways in: a folder elsewhere, and the library folder itself. +describe('ImportHints', () => { + it('links to the Import page views, not to Settings tabs', () => { + render() + expect(screen.getByRole('link', { name: 'Import from a folder' })).toHaveAttribute('href', '/import?view=folder') + expect(screen.getByRole('link', { name: 'Review my library' })).toHaveAttribute('href', '/import') + }) +}) diff --git a/web/src/components/ImportHints.tsx b/web/src/components/ImportHints.tsx index 40ad824d0..52ec5078e 100644 --- a/web/src/components/ImportHints.tsx +++ b/web/src/components/ImportHints.tsx @@ -3,10 +3,9 @@ import { useTranslation } from 'react-i18next' import { btn, btnSize } from './buttons' // Discoverability CTA shown on empty Queue/Wanted states (#1184). New users who -// already have files on disk often don't realise Manual Import / Scan Library -// exist, so we point them straight at the existing flows rather than building a -// new import UI. Both targets are Settings sub-tabs (deep-linked via ?tab=…): -// Manual Import lives under Import / Migrate, Scan Library under General. +// already have files on disk often don't realise Bindery can take them in, so +// we point them straight at the Import page: its folder view for files +// elsewhere, its library view for files already in the library folder. export default function ImportHints() { const { t } = useTranslation() return ( @@ -18,10 +17,10 @@ export default function ImportHints() { {t('importHints.body')}

- + {t('importHints.manualImport')} - + {t('importHints.scanLibrary')}
diff --git a/web/src/components/SortHeader.tsx b/web/src/components/SortHeader.tsx new file mode 100644 index 000000000..04da6527e --- /dev/null +++ b/web/src/components/SortHeader.tsx @@ -0,0 +1,54 @@ +import { useTranslation } from 'react-i18next' + +interface Props { + label: string + // The sort key this column sorts by. + sortKey: K + // The active sort key and direction. + activeKey: K + direction: 'asc' | 'desc' + // Clicking the active column flips direction; another column starts at its + // natural direction. + onSort: (key: K) => void + className?: string +} + +// Chevrons drawn as SVG in the same outline style as the header icons, so the +// arrow never depends on a font having the glyph. +function SortIcon({ state }: { state: 'asc' | 'desc' | 'none' }) { + return ( + + ) +} + +// SortHeader is a clickable column header showing the active sort. BooksPage +// and AuthorDetailPage each carry their own copy; this is the third, so it +// lives here, and moving those two onto it is a separate change. +// +// Tailwind v4's Preflight gives + + ) +} diff --git a/web/src/components/import/BookPicker.tsx b/web/src/components/import/BookPicker.tsx new file mode 100644 index 000000000..fcc08488d --- /dev/null +++ b/web/src/components/import/BookPicker.tsx @@ -0,0 +1,95 @@ +import { forwardRef, useEffect, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { api } from '../../api/client' +import type { Book } from '../../api/client' + +interface Props { + onPick: (b: Book) => void + onCancel?: () => void + // Prefills the search, so a person corrects a query instead of typing one. + initialTerm?: string + // Marks one result as the current choice (the adoption editor's radio rows). + selectedId?: number | null +} + +// BookPicker is a debounced search over the EXISTING catalogue (the same +// /book?search= endpoint FixMatchModal uses). It resolves a file to a book +// already in the library. A book that is not in the library at all is +// CatalogueAdder's job. Only the local catalogue is searched here, never a +// metadata provider. +const BookPicker = forwardRef(function BookPicker({ onPick, onCancel, initialTerm = '', selectedId }, ref) { + const { t } = useTranslation() + const [term, setTerm] = useState(initialTerm) + const [results, setResults] = useState([]) + const [loading, setLoading] = useState(false) + + useEffect(() => { + const q = term.trim() + if (q.length < 2) { + setResults([]) + return + } + let cancelled = false + setLoading(true) + const handle = setTimeout(async () => { + try { + const { items } = await api.listBooks({ search: q, limit: 20 }) + if (!cancelled) setResults(items) + } catch { + if (!cancelled) setResults([]) + } finally { + if (!cancelled) setLoading(false) + } + }, 300) + return () => { cancelled = true; clearTimeout(handle) } + }, [term]) + + return ( +
+
+ setTerm(e.target.value)} + placeholder={t('manualImport.searchPlaceholder', 'Search your library for the book…')} + aria-label={t('manualImport.searchLabel', 'Search your library')} + className="flex-1 px-2 py-1 rounded border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-xs" + /> + {onCancel && ( + + )} +
+
+ {loading &&

{t('common.loading', 'Loading...')}

} + {!loading && term.trim().length >= 2 && results.length === 0 && ( +

+ {t('manualImport.noResults', 'No matching books in your library.')} +

+ )} + {results.map(b => ( + + ))} +
+
+ ) +}) + +export default BookPicker diff --git a/web/src/components/import/CatalogueAdder.tsx b/web/src/components/import/CatalogueAdder.tsx new file mode 100644 index 000000000..5505089bb --- /dev/null +++ b/web/src/components/import/CatalogueAdder.tsx @@ -0,0 +1,146 @@ +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { resolveBookQuery } from '../../api/booklookup' +import type { Book } from '../../api/client' +import { btn, btnSize } from '../buttons' + +interface Props { + // Prefilled from what the scan read out of the file. + initialQuery: string + // Adds the chosen result somehow (add then import, or adopt). A thrown + // error is shown inline. + onChoose: (b: Book) => Promise + // Sentence under the search box saying what choosing a result does. + hint?: string + actionLabel?: string + busyLabel?: string +} + +// CatalogueAdder resolves the case BookPicker cannot: the file is for a book +// that is not in the library at all, so there is no row to bind it to (#1719). +// It searches provider metadata (the same ISBN / ASIN / free-text dispatch the +// Add Book modal uses) and hands the chosen result to onChoose. +// +// It starts collapsed, and a provider is asked only when the person submits +// the search. Opening it, typing and rendering never spend provider quota. +export default function CatalogueAdder({ initialQuery, onChoose, hint, actionLabel, busyLabel }: Props) { + const { t } = useTranslation() + const [open, setOpen] = useState(false) + const [term, setTerm] = useState(initialQuery) + const [results, setResults] = useState([]) + const [searching, setSearching] = useState(false) + const [searched, setSearched] = useState(false) + const [adding, setAdding] = useState('') + const [error, setError] = useState('') + + if (!open) { + return ( + + ) + } + + const search = async () => { + const q = term.trim() + if (!q) return + setSearching(true) + setError('') + try { + setResults(await resolveBookQuery(q)) + setSearched(true) + } catch (e) { + setResults([]) + setError(e instanceof Error ? e.message : 'Metadata search failed') + } finally { + setSearching(false) + } + } + + const choose = async (b: Book) => { + if (!b.foreignBookId || !b.author?.authorName) return + setAdding(b.foreignBookId) + setError('') + try { + await onChoose(b) + } catch (e) { + setError(e instanceof Error ? e.message : 'Could not add the book') + } finally { + setAdding('') + } + } + + return ( +
+
{ e.preventDefault(); void search() }} + > + setTerm(e.target.value)} + placeholder={t('manualImport.metadataPlaceholder', 'Title, author, ISBN, or ASIN')} + aria-label={t('manualImport.metadataLabel', 'Search metadata')} + className="flex-1 px-2 py-1 rounded border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-xs" + /> + + +
+

+ {hint ?? t('manualImport.metadataHint', 'Adds the book to your library and links this file to it. No indexer search is started.')} +

+ {error &&

{error}

} +
+ {!searching && searched && results.length === 0 && !error && ( +

+ {t('manualImport.metadataNoResults', 'No metadata results for that search.')} +

+ )} + {results.map(b => { + // An author NAME is enough; the backend resolves a missing author id. + // Without one there is nothing to create the book under. + const canAdd = Boolean(b.author?.authorName) + return ( +
+
+ {b.title} + {b.author?.authorName && ( + · {b.author.authorName} + )} +
+ +
+ ) + })} +
+
+ ) +} diff --git a/web/src/components/useUnmatchedCount.ts b/web/src/components/useUnmatchedCount.ts new file mode 100644 index 000000000..fe6a8eb6c --- /dev/null +++ b/web/src/components/useUnmatchedCount.ts @@ -0,0 +1,33 @@ +import { useEffect, useState } from 'react' +import { api } from '../api/client' + +// The count behind the Import nav badge: books the library scan could not +// match that still need a decision. Read once per app load for an admin, then +// updated from the event the adoption view fires after each adopt, ignore or +// scan (P5). Nothing polls. + +const EVENT = 'bindery:unmatched-changed' + +export function notifyUnmatchedChanged(pending: number) { + window.dispatchEvent(new CustomEvent(EVENT, { detail: pending })) +} + +export function useUnmatchedCount(enabled: boolean): number { + const [count, setCount] = useState(0) + + useEffect(() => { + if (!enabled) return + let cancelled = false + api.unmatchedSummary() + .then(s => { if (!cancelled) setCount(s.pending) }) + .catch(() => { /* the badge is a convenience; no count shows nothing */ }) + const onChange = (e: Event) => setCount((e as CustomEvent).detail) + window.addEventListener(EVENT, onChange) + return () => { + cancelled = true + window.removeEventListener(EVENT, onChange) + } + }, [enabled]) + + return enabled ? count : 0 +} diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index 612933617..ddadbae66 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -80,7 +80,9 @@ "blocklist": "Blocklist", "settings": "Settings", "users": "Users", - "updateAvailable": "Update available" + "updateAvailable": "Update available", + "importUnmatched_one": "{{count}} book needs a decision", + "importUnmatched_other": "{{count}} books need a decision" }, "librarySearch": { "label": "Search library", @@ -128,7 +130,7 @@ "failed": "Failed: {{error}}", "searchLabel": "Search your library", "searchPlaceholder": "Search your library for the book…", - "noResults": "No matching books in your library. Search metadata below to add it.", + "noResults": "No matching books in your library.", "metadataOpen": "Not in your library? Search metadata", "metadataLabel": "Search metadata", "metadataPlaceholder": "Title, author, ISBN, or ASIN", @@ -140,6 +142,159 @@ "metadataAdd": "Add and use", "metadataAdding": "Adding…" }, + "importPage": { + "title": "Import", + "subtitle": "Bring books you already have into Bindery.", + "viewsLabel": "Import views", + "viewLibrary": "In your library", + "viewFolder": "From a folder" + }, + "adoption": { + "adminOnly": "Only an admin can decide what the files in the library are.", + "searchPlaceholder": "Search titles, authors and folders", + "searchLabel": "Search books to decide", + "stateLabel": "Show", + "state": { + "pending": "Needs a decision", + "ignored": "Ignored", + "adopted": "Adopted" + }, + "filterReason": "Why it did not match", + "filterFormat": "Format", + "chipFolder": "Folder: {{folder}}", + "reason": { + "author_not_in_library": "Author not in your library", + "no_candidate_books": "No book waiting for a file", + "no_title_match": "No close title", + "no_title_parsed": "No title read" + }, + "confirm": "Confirm", + "choose": "Choose book", + "ignore": "Ignore", + "unignore": "Unignore", + "undo": "Undo", + "hint": { + "reasonTooltip": "Scan reason: {{code}}", + "authorMissing": "{{author}} is not in your library yet. Add the author, then scan again.", + "authorUnreadable": "No author could be read from these files. Choose the book they are.", + "noCandidates": "{{author}} has no book in your library waiting for these files. Choose the book or search metadata.", + "noTitle": "No title could be read from the file names or tags. Choose the book they are.", + "noTitleMatch": "No book by {{author}} has a title close to this one. Choose the book it is.", + "noTitleMatchNoAuthor": "No book in your library has a title close to this one. Choose the book it is.", + "strong": "Strong match: {{title}} by {{author}}. Confirm it or choose another book.", + "possible": "Possible match: {{title}} by {{author}}, {{percent}}% title similarity. Check it, then adopt it or choose another book." + }, + "row": { + "files_one": "{{count}} file", + "files_other": "{{count}} files", + "adoptedAs": "Adopted as" + }, + "outcome": { + "adopting": "Adopting as {{title}}…", + "adopted": "Adopted as {{title}} by {{author}}.", + "created": "Added to your library, unmonitored.", + "ignored": "Ignored. Later scans keep it out of this list.", + "undoing": "Undoing…", + "restored": "Back in Needs a decision.", + "keptBook": "Files removed. The book stayed because it is now in use." + }, + "editor": { + "heading": "Which book is {{name}}?", + "suggestions": "Suggestions", + "score": "{{percent}}% title match", + "fromSearch": "From search", + "searchLibrary": "Search your library", + "metadataHint": "Adds the book to your library unmonitored, as the format of these files, and adopts them. Nothing is downloaded.", + "addAndAdopt": "Add and adopt", + "adding": "Adding…", + "files_one": "{{count}} file", + "files_other": "{{count}} files", + "moreFiles": "and {{count}} more", + "inPlace": "Files stay where they are. Picking a book already in your library leaves its owner and monitoring as they are.", + "adoptAs": "Adopt as {{title}}", + "pickFirst": "Pick a book" + }, + "col": { + "book": "Book", + "match": "Best match", + "actions": "Actions" + }, + "caption": { + "pending": "Books in your library that need a decision", + "ignored": "Ignored books in your library", + "adopted": "Books you adopted from your library", + "keys": "Arrow keys move between books, Enter opens one, i ignores, u undoes." + }, + "rail": { + "label": "Folders with the most books to decide", + "heading": "Folders", + "notInLibrary": "{{author}} is not in your library", + "ignore": "Ignore folder", + "all": "All folders" + }, + "ignoreFolder": { + "title": "Ignore every book in {{folder}}?", + "body": "They leave this list and later scans keep them out. You can unignore any of them from the Ignored list.", + "confirm": "Ignore folder" + }, + "summary": { + "label": "Library scan summary", + "scanning": "Scanning your library…", + "none": "Nothing needs a decision", + "nudge": "Author added. Scan now to match their files.", + "scanningButton": "Scanning…", + "scanNow": "Scan now", + "scanError": "The last scan did not finish: {{error}}", + "noFiles": "The last scan found no book files. Check that the library folder is mounted, then scan again. Your decisions were kept.", + "truncated": "This library is larger than one scan lists. Adopt or ignore some books and scan again to see the rest.", + "needDecision_one": "book needs a decision", + "needDecision_other": "books need a decision", + "files_one": "file", + "files_other": "files", + "never": "never", + "lastScanLabel": "last scan", + "adoptedLabel": "adopted", + "ignoredLabel": "ignored", + "justNow": "just now" + }, + "empty": { + "filteredTitle": "No books match these filters", + "filteredBody": "Try another search, or clear the filters to see every book.", + "clear": "Clear filters", + "ignoredTitle": "Nothing is ignored", + "ignoredBody": "Books you ignore land here, and you can bring any of them back.", + "adoptedTitle": "Nothing adopted yet", + "adoptedBody": "Books you adopt stay here for 30 days, with Undo.", + "neverTitle": "Your library has not been scanned yet", + "neverBody": "A scan looks through your library folders and lists every book it cannot match to your library, so you can decide what each one is. Files are never moved.", + "scanLibrary": "Scan library", + "doneTitle": "Every book in your library is matched", + "doneBody": "Nothing needs a decision. New files show up here after the next scan." + }, + "addAuthor": "Add author", + "showFiles": "Show files", + "more": "More", + "moreFor": "More actions for {{name}}", + "match": { + "strong": "Strong match", + "possible": "Possible match" + }, + "short": { + "authorMissing": "{{author}} is not in your library", + "authorUnreadable": "No author could be read", + "noCandidates": "No book by {{author}} is waiting for a file", + "noTitle": "No title could be read", + "noTitleMatch": "No close title by {{author}}", + "noTitleMatchNoAuthor": "No close title in your library" + }, + "group": { + "label": "{{author}}, {{count}} books", + "counts": "{{count}} books, {{files}} files", + "sentence": "Author not in your library yet", + "show": "Show books", + "hide": "Hide books" + } + }, "search": { "heading": "Search Indexers", "placeholder": "Title, author, or keyword…", @@ -511,8 +666,8 @@ "importHints": { "heading": "Already have files on disk?", "body": "Bindery only auto-imports downloads it grabbed itself. Point it at files you already have, or rescan your library folder.", - "manualImport": "Import them", - "scanLibrary": "Scan Library" + "manualImport": "Import from a folder", + "scanLibrary": "Review my library" }, "blocklist": { "title": "Blocklist", @@ -874,20 +1029,11 @@ "unmatched": "Unmatched:", "alreadyTracked": "Already tracked:", "scannedPaths": "Scanned:", + "unmatchedUnits_one": "{{count}} book needs a decision.", + "unmatchedUnits_other": "{{count}} books need a decision.", + "reviewUnmatched_one": "Review {{count}} unmatched book", + "reviewUnmatched_other": "Review {{count}} unmatched books", "scanNoFilesWarning": "No book files found under {{path}} — check BINDERY_LIBRARY_DIR points at your library.", - "scanAllUnmatchedHint": "Files were found but none matched a book. Unmatched files need their author's book catalogue populated first — refresh the author (or re-run the import so it fetches books) and scan again.", - "scanUnmatchedAuthorUnknown": "Files were found but none matched a book. The parsed author \"{{author}}\" doesn't match any author in your library, so no book could be considered — check the files' embedded tags and folder names against the Parsed Author column below. Refreshing an author won't help here.", - "scanUnmatchedAuthorsUnknown": "Files were found but none matched a book. None of their parsed authors match an author in your library, so no book could be considered — compare the Parsed Author column below with your authors' names. Refreshing an author won't help here.", - "scanReason": { - "author_not_in_library": "Parsed author isn't in your library", - "no_candidate_books": "Author matched, but has no book waiting for a file", - "no_title_match": "No book by that author matched this title", - "no_title_parsed": "No title could be read from this file's name or tags" - }, - "scanUnmatchedPath": "Path", - "scanUnmatchedParsedTitle": "Parsed Title", - "scanUnmatchedParsedAuthor": "Parsed Author", - "scanUnmatchedWhy": "Why", "opds": "OPDS Feed", "opdsFeedUrl": "OPDS Feed URL", "opdsHint": "Use this URL in KOReader, Moon+ Reader, or any OPDS client. Authenticate with your API key as the password (leave the username blank or use any value).", @@ -1026,7 +1172,10 @@ "goodreadsCommitResult": "{{added}} added · {{skipped}} skipped · {{failed}} failed", "goodreadsNoResolved": "No rows could be resolved — nothing to import. Try widening the shelf filter or check the unmatched rows below.", "goodreadsUploadFailed": "Upload failed", - "configureHardcoverToken": "Configure the Hardcover API token in API Keys settings →" + "configureHardcoverToken": "Configure the Hardcover API token in API Keys settings →", + "folderMovedHeading": "Import from a folder", + "folderMovedBody": "Scanning a folder and importing what it matches is on the Import page now, beside the books a library scan could not match.", + "folderMovedLink": "Open Import" }, "oidc": { "heading": "SSO / OIDC Providers", diff --git a/web/src/i18n/locales/ko.json b/web/src/i18n/locales/ko.json index 895f02527..bffb7b366 100644 --- a/web/src/i18n/locales/ko.json +++ b/web/src/i18n/locales/ko.json @@ -593,7 +593,6 @@ "unmatched": "미일치:", "scannedPaths": "스캔 경로:", "scanNoFilesWarning": "{{path}} 아래에서 도서 파일을 찾을 수 없습니다 — BINDERY_LIBRARY_DIR이 라이브러리를 가리키는지 확인하세요.", - "scanAllUnmatchedHint": "파일은 발견되었으나 일치하는 도서가 없습니다. 미일치 파일은 해당 저자의 도서 목록이 먼저 채워져야 합니다. 저자를 새로고침하거나(또는 가져오기를 다시 실행해 도서를 가져온 후) 다시 스캔하세요.", "opds": "OPDS 피드", "opdsFeedUrl": "OPDS 피드 URL", "opdsHint": "KOReader, Moon+ Reader 또는 OPDS 클라이언트에서 이 URL을 사용하세요. API 키를 비밀번호로 입력해 인증하세요(사용자 이름은 비워 두거나 임의 값을 입력하세요).", diff --git a/web/src/pages/ManualImportPage.tsx b/web/src/pages/ManualImportPage.tsx deleted file mode 100644 index 023f2b5a6..000000000 --- a/web/src/pages/ManualImportPage.tsx +++ /dev/null @@ -1,618 +0,0 @@ -import { useEffect, useMemo, useState } from 'react' -import { useTranslation } from 'react-i18next' -import { api } from '../api/client' -import { resolveBookQuery } from '../api/booklookup' -import type { BatchImportItem, BatchImportResult, Book, ScanItem } from '../api/client' -import { btn, btnSize } from '../components/buttons' - -// ManualImportPage is the manual-import wizard (#1236). It scans a folder with -// the recursive bulk-import scan (#1434), groups the discovered units by match -// confidence, and lets the user resolve each one before importing: -// - confident: a preselected catalogue match, deselectable and overridable; -// - ambiguous: a candidate picker over the returned catalogue candidates; -// - none: a search box to bind the unit to an EXISTING catalogue book, plus -// a metadata search that creates the book (and its author) when the library -// has no row to bind to (#1719). -// Resolved units are imported (per-unit or in bulk) through the batch endpoint, -// which reports per-item success/failure. - -const MATCH_ORDER: ScanItem['match'][] = ['confident', 'ambiguous', 'none'] - -// RowState is the per-unit resolution the user is building, keyed by unit path. -interface RowState { - // chosen is the catalogue book the unit will import against (a confident - // match, a picked candidate, or a searched existing book). null until resolved. - chosen: Book | null - // format override: '' = auto-detect, or 'ebook' / 'audiobook'. Defaults to the - // scan's detected format. - format: string - // selected marks the unit for the next bulk import. Only meaningful when chosen. - selected: boolean -} - -function bookLabel(b: Book): string { - return b.author?.authorName ? `${b.title} · ${b.author.authorName}` : b.title -} - -function groupHeading(match: ScanItem['match']): string { - switch (match) { - case 'confident': return 'Matched' - case 'ambiguous': return 'Needs a choice' - default: return 'Unmatched' - } -} - -export default function ManualImportPage() { - const { t } = useTranslation() - const [path, setPath] = useState('') - const [scanning, setScanning] = useState(false) - const [items, setItems] = useState(null) - const [truncated, setTruncated] = useState(false) - const [rows, setRows] = useState>({}) - const [results, setResults] = useState>({}) - const [importingPaths, setImportingPaths] = useState>(() => new Set()) - const [scanError, setScanError] = useState('') - const [importError, setImportError] = useState('') - - useEffect(() => { - document.title = 'Manual Import · Bindery' - return () => { document.title = 'Bindery' } - }, []) - - const handleScan = async () => { - const p = path.trim() - if (!p) return - setScanning(true) - setScanError('') - setImportError('') - setItems(null) - setResults({}) - try { - const r = await api.scanFolder(p) - setItems(r.items) - setTruncated(r.truncated) - const init: Record = {} - for (const it of r.items) { - const chosen = it.match === 'confident' && it.book ? it.book : null - init[it.path] = { chosen, format: it.detectedFormat || '', selected: Boolean(chosen) } - } - setRows(init) - } catch (e) { - setScanError(e instanceof Error ? e.message : 'Scan failed') - } finally { - setScanning(false) - } - } - - const patchRow = (unitPath: string, patch: Partial) => - setRows(prev => ({ ...prev, [unitPath]: { ...prev[unitPath], ...patch } })) - - const pickBook = (unitPath: string, book: Book) => - patchRow(unitPath, { chosen: book, selected: true }) - - const toggleSelected = (unitPath: string) => - setRows(prev => { - const r = prev[unitPath] - if (!r || !r.chosen) return prev - return { ...prev, [unitPath]: { ...r, selected: !r.selected } } - }) - - // A path is submittable when it has a chosen book, is selected, and hasn't - // already been accepted by a prior import. A `none` unit the user never - // resolved has chosen=null, so it is never submitted. - const submittablePaths = useMemo(() => { - if (!items) return [] - return items - .map(it => it.path) - .filter(p => { - const r = rows[p] - return Boolean(r?.chosen) && r.selected && !results[p]?.accepted - }) - }, [items, rows, results]) - - const submit = async (paths: string[]) => { - const batch: BatchImportItem[] = [] - for (const p of paths) { - const r = rows[p] - if (!r?.chosen || !r.selected) continue - if (results[p]?.accepted) continue - batch.push({ path: p, bookId: r.chosen.id, format: r.format || undefined }) - } - if (batch.length === 0) return - setImportError('') - setImportingPaths(prev => { const n = new Set(prev); batch.forEach(b => n.add(b.path)); return n }) - try { - const res = await api.batchImport(batch) - setResults(prev => { - const n = { ...prev } - for (const r of res.results) n[r.path] = r - return n - }) - } catch (e) { - setImportError(e instanceof Error ? e.message : 'Import failed') - } finally { - setImportingPaths(prev => { const n = new Set(prev); batch.forEach(b => n.delete(b.path)); return n }) - } - } - - const allSelected = submittablePaths.length > 0 && - submittablePaths.every(p => rows[p]?.selected) - - const toggleSelectAll = () => { - if (!items) return - // Resolved, not-yet-imported units only — a `none` unit with no book can't - // be selected. - const resolvable = items - .map(it => it.path) - .filter(p => rows[p]?.chosen && !results[p]?.accepted) - const turnOn = !resolvable.every(p => rows[p]?.selected) - setRows(prev => { - const n = { ...prev } - for (const p of resolvable) n[p] = { ...n[p], selected: turnOn } - return n - }) - } - - const groups = useMemo(() => { - const g: Record = { confident: [], ambiguous: [], none: [] } - for (const it of items ?? []) g[it.match].push(it) - return g - }, [items]) - - return ( -
-
-

- {t('manualImport.title', 'Manual Import')} -

-

- {t('manualImport.description', 'Scan a folder of files already on disk, match each book to your library, and import them. A file with no match can be added from a metadata search.')} -

-
- -
- setPath(e.target.value)} - onKeyDown={e => { if (e.key === 'Enter') handleScan() }} - placeholder={t('manualImport.pathPlaceholder', '/downloads/books')} - aria-label={t('manualImport.pathLabel', 'Folder to scan')} - className="flex-1 px-3 py-2 rounded-md border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-sm" - /> - -
- - {scanError && ( -

{scanError}

- )} - - {items && items.length === 0 && ( -

- {t('manualImport.empty', 'No book files or folders found here.')} -

- )} - - {items && items.length > 0 && ( - <> - {truncated && ( -

- {t('manualImport.truncated', 'Showing the first 1000 units; narrow the folder to see the rest.')} -

- )} - - {/* Select-all / bulk import bar */} -
- - -
- - {importError && ( -

{importError}

- )} - - {MATCH_ORDER.map(match => { - const group = groups[match] - if (group.length === 0) return null - return ( -
-

- {t(`manualImport.group.${match}`, groupHeading(match))} ({group.length}) -

-
- {group.map(it => ( - pickBook(it.path, b)} - onToggle={() => toggleSelected(it.path)} - onFormat={f => patchRow(it.path, { format: f })} - onImport={() => submit([it.path])} - /> - ))} -
-
- ) - })} - - )} -
- ) -} - -interface ImportRowProps { - item: ScanItem - row: RowState | undefined - result: BatchImportResult | undefined - importing: boolean - onPick: (b: Book) => void - onToggle: () => void - onFormat: (f: string) => void - onImport: () => void -} - -function ImportRow({ item, row, result, importing, onPick, onToggle, onFormat, onImport }: ImportRowProps) { - const { t } = useTranslation() - const [overriding, setOverriding] = useState(false) - const chosen = row?.chosen ?? null - const accepted = result?.accepted - - const badgeCls: Record = { - confident: 'bg-emerald-100 dark:bg-emerald-950 text-emerald-700 dark:text-emerald-400', - ambiguous: 'bg-amber-100 dark:bg-amber-950 text-amber-700 dark:text-amber-400', - none: 'bg-slate-200 dark:bg-zinc-800 text-slate-600 dark:text-zinc-400', - } - - return ( -
-
- -
-
- {item.name} - - {t(`manualImport.group.${item.match}`, groupHeading(item.match))} - - - {item.detectedFormat} - -
- {/* Full source path so the user can tell which file each row refers to; - the basename alone is ambiguous across folders (#1435). */} -
{item.path}
-
- {t('manualImport.parsed', { title: item.parsedTitle || '?', author: item.parsedAuthor || '?', defaultValue: `parsed: ${item.parsedTitle || '?'} / ${item.parsedAuthor || '?'}` })} -
- - {/* Resolution area */} -
- {chosen && !overriding && ( -
- → {bookLabel(chosen)} - {!accepted && ( - - )} -
- )} - - {/* Ambiguous candidate picker */} - {item.match === 'ambiguous' && !chosen && item.candidates && item.candidates.length > 0 && ( -
- {t('manualImport.pickCandidate', 'Pick the correct book')} - {item.candidates.map(c => ( - - ))} -
- )} - - {/* None / override: search the existing catalogue, or add the book - from provider metadata when the library has nothing to bind to. */} - {((item.match === 'none' && !chosen) || overriding) && ( - <> - { onPick(b); setOverriding(false) }} - onCancel={overriding ? () => setOverriding(false) : undefined} - /> - { onPick(b); setOverriding(false) }} - /> - - )} -
-
- -
- - -
-
- - {result && ( -

- {accepted - ? t('manualImport.queued', 'Queued — importing in the background; watch the Queue for progress.') - : t('manualImport.failed', { error: result.error || 'failed', defaultValue: `Failed: ${result.error || 'failed'}` })} -

- )} -
- ) -} - -// BookPicker is a debounced search over the EXISTING catalogue (reuses the same -// /book?search= endpoint FixMatchModal uses). It resolves a `none` unit, or an -// override on a matched one, to a book already in the library. A unit whose -// book is not in the library at all is CatalogueAdder's job, rendered directly -// below this picker. -function BookPicker({ onPick, onCancel }: { onPick: (b: Book) => void; onCancel?: () => void }) { - const { t } = useTranslation() - const [term, setTerm] = useState('') - const [results, setResults] = useState([]) - const [loading, setLoading] = useState(false) - - useEffect(() => { - const q = term.trim() - if (q.length < 2) { - setResults([]) - return - } - let cancelled = false - setLoading(true) - const handle = setTimeout(async () => { - try { - const { items } = await api.listBooks({ search: q, limit: 20 }) - if (!cancelled) setResults(items) - } catch { - if (!cancelled) setResults([]) - } finally { - if (!cancelled) setLoading(false) - } - }, 300) - return () => { cancelled = true; clearTimeout(handle) } - }, [term]) - - return ( -
-
- setTerm(e.target.value)} - placeholder={t('manualImport.searchPlaceholder', 'Search your library for the book…')} - aria-label={t('manualImport.searchLabel', 'Search your library')} - className="flex-1 px-2 py-1 rounded border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-xs" - /> - {onCancel && ( - - )} -
-
- {loading &&

{t('common.loading', 'Loading...')}

} - {!loading && term.trim().length >= 2 && results.length === 0 && ( -

- {t('manualImport.noResults', 'No matching books in your library. Search metadata below to add it.')} -

- )} - {results.map(b => ( - - ))} -
-
- ) -} - -// CatalogueAdder resolves the case BookPicker cannot: the file is for a book -// that is not in the library at all, so there is no row to bind it to (#1719). -// It searches provider metadata (the same ISBN / ASIN / free-text dispatch the -// Add Book modal uses) and creates the picked result through POST /author/book, -// which creates the author too when it is new. The created book is handed back -// so the unit resolves like any other and imports through the existing batch -// call. -// -// searchOnAdd is deliberately false: the user is importing a file they already -// have, so kicking off an indexer search for it would grab a second copy. -function CatalogueAdder({ item, onAdded }: { item: ScanItem; onAdded: (b: Book) => void }) { - const { t } = useTranslation() - const [open, setOpen] = useState(false) - // Prefilled from what the scan already read out of the file, so the user is - // correcting a query rather than retyping one. - const [term, setTerm] = useState(() => [item.parsedTitle, item.parsedAuthor].filter(Boolean).join(' ')) - const [results, setResults] = useState([]) - const [searching, setSearching] = useState(false) - const [searched, setSearched] = useState(false) - const [adding, setAdding] = useState('') - const [error, setError] = useState('') - - if (!open) { - return ( - - ) - } - - const search = async () => { - const q = term.trim() - if (!q) return - setSearching(true) - setError('') - try { - setResults(await resolveBookQuery(q)) - setSearched(true) - } catch (e) { - setResults([]) - setError(e instanceof Error ? e.message : 'Metadata search failed') - } finally { - setSearching(false) - } - } - - const add = async (b: Book) => { - if (!b.foreignBookId || !b.author?.authorName) return - setAdding(b.foreignBookId) - setError('') - try { - const created = await api.addBook({ - foreignBookId: b.foreignBookId, - // May be empty (e.g. a Google Books or DNB result carries a name but no - // author id); the backend resolves the author by name in that case. - foreignAuthorId: b.author.foreignAuthorId ?? '', - authorName: b.author.authorName, - searchOnAdd: false, - }) - onAdded(created) - } catch (e) { - setError(e instanceof Error ? e.message : 'Could not add the book') - } finally { - setAdding('') - } - } - - return ( -
-
- setTerm(e.target.value)} - onKeyDown={e => { if (e.key === 'Enter') search() }} - placeholder={t('manualImport.metadataPlaceholder', 'Title, author, ISBN, or ASIN')} - aria-label={t('manualImport.metadataLabel', 'Search metadata')} - className="flex-1 px-2 py-1 rounded border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-xs" - /> - - -
-

- {t('manualImport.metadataHint', 'Adds the book to your library and links this file to it. No indexer search is started.')} -

- {error &&

{error}

} -
- {!searching && searched && results.length === 0 && !error && ( -

- {t('manualImport.metadataNoResults', 'No metadata results for that search.')} -

- )} - {results.map(b => { - // An author NAME is enough; the backend resolves a missing author id. - // Without one there is nothing to create the book under. - const canAdd = Boolean(b.author?.authorName) - return ( -
-
- {b.title} - {b.author?.authorName && ( - · {b.author.authorName} - )} -
- -
- ) - })} -
-
- ) -} diff --git a/web/src/pages/QueuePage.test.tsx b/web/src/pages/QueuePage.test.tsx index 8c65d8cf6..2384a0574 100644 --- a/web/src/pages/QueuePage.test.tsx +++ b/web/src/pages/QueuePage.test.tsx @@ -191,9 +191,9 @@ describe('QueuePage', () => { expect(await screen.findByText('Queue is empty')).toBeInTheDocument() const manualImport = screen.getByRole('link', { name: 'Import them' }) - expect(manualImport).toHaveAttribute('href', '/settings?tab=import') + expect(manualImport).toHaveAttribute('href', '/import?view=folder') const scanLibrary = screen.getByRole('link', { name: 'Scan Library' }) - expect(scanLibrary).toHaveAttribute('href', '/settings?tab=general') + expect(scanLibrary).toHaveAttribute('href', '/import') }) it('renders queue statuses, progress, fallback errors, and error prefixes', async () => { diff --git a/web/src/pages/SettingsPage.test.tsx b/web/src/pages/SettingsPage.test.tsx index 391fdcc40..cc8652d22 100644 --- a/web/src/pages/SettingsPage.test.tsx +++ b/web/src/pages/SettingsPage.test.tsx @@ -775,12 +775,16 @@ describe('SettingsPage', () => { expect(screen.getByText('settings.general.scanNoFilesWarning')).toBeInTheDocument() }) - it('hints to populate catalogue when files found but none matched', async () => { + it('counts unmatched books and links to the Import page instead of listing files', async () => { + // The per file table and its reason hints moved to the Import page's + // "In your library" view, where each book can be acted on. Settings keeps + // the count and the way there. vi.mocked(api.libraryScanStatus).mockResolvedValue({ ran_at: new Date().toISOString(), - files_found: 12, + files_found: 212, reconciled: 0, - unmatched: 12, + unmatched: 212, + unmatched_units: 14, library_dir: '/books', scanned_paths: ['/books'], no_files_found: false, @@ -789,91 +793,24 @@ describe('SettingsPage', () => { renderSettings() expect(await screen.findByText('settings.general.lastScan')).toBeInTheDocument() - expect(screen.getByText('settings.general.scanAllUnmatchedHint')).toBeInTheDocument() - // The no-files warning must NOT appear when files were found. + expect(screen.getByRole('link', { name: /settings\.general\.reviewUnmatched/ })).toHaveAttribute('href', '/import') + expect(screen.queryByRole('table')).not.toBeInTheDocument() expect(screen.queryByText('settings.general.scanNoFilesWarning')).not.toBeInTheDocument() }) - it('names the parsed author instead of blaming the catalogue when no author matched', async () => { - // #1958: the catalogue hint sent a user author-refreshing for two weeks - // while their real problem was the file's tags. When every unmatched file - // reports author_not_in_library, the hint must name the parsed author and - // the old advice must not be shown. + it('shows no review link when nothing needs a decision', async () => { vi.mocked(api.libraryScanStatus).mockResolvedValue({ ran_at: new Date().toISOString(), - files_found: 1, - reconciled: 0, - unmatched: 1, - library_dir: '/books', - scanned_paths: ['/books'], - no_files_found: false, - unmatched_files: [{ - path: '/books/Álvaro Enrigue/You Dreamed of Empires/You Dreamed of Empires.m4b', - parsed_title: 'You Dreamed of Empires', - parsed_author: 'Álvaro Enrigue, Natasha Wimmer - translator, Gabriel Porras', - reason: 'author_not_in_library', - }], - }) - - renderSettings() - - expect(await screen.findByText('settings.general.lastScan')).toBeInTheDocument() - expect(screen.getByText('settings.general.scanUnmatchedAuthorUnknown')).toBeInTheDocument() - expect(screen.queryByText('settings.general.scanAllUnmatchedHint')).not.toBeInTheDocument() - // The per-file diagnosis is rendered alongside the parsed values. - expect(screen.getByText('settings.general.scanReason.author_not_in_library')).toBeInTheDocument() - }) - - it('does not generalise the author hint from a truncated unmatched sample', async () => { - // The scanner caps unmatched_files at 1000 entries. every() over that sample - // proves nothing about the other 4000, so the specific "your authors don't - // match" hint must fall back to the generic advice when the list is capped. - vi.mocked(api.libraryScanStatus).mockResolvedValue({ - ran_at: new Date().toISOString(), - files_found: 5000, - reconciled: 0, - unmatched: 5000, - library_dir: '/books', - scanned_paths: ['/books'], - no_files_found: false, - unmatched_files: [{ - path: '/books/Becky Chambers/A Psalm for the Wild-Built.epub', - parsed_title: 'A Psalm for the Wild-Built', - parsed_author: 'Becky Chambers', - reason: 'author_not_in_library', - }], - }) - - renderSettings() - - expect(await screen.findByText('settings.general.lastScan')).toBeInTheDocument() - expect(screen.getByText('settings.general.scanAllUnmatchedHint')).toBeInTheDocument() - expect(screen.queryByText('settings.general.scanUnmatchedAuthorUnknown')).not.toBeInTheDocument() - }) - - it('keeps the catalogue hint when the author matched but has no candidate books', async () => { - // The original advice is right for #875's case, so a no_candidate_books - // diagnosis must still produce it — the fix narrows the message, it doesn't - // replace it. - vi.mocked(api.libraryScanStatus).mockResolvedValue({ - ran_at: new Date().toISOString(), - files_found: 2, - reconciled: 0, - unmatched: 2, - library_dir: '/books', - scanned_paths: ['/books'], - no_files_found: false, - unmatched_files: [ - { path: '/books/Andy Weir/Project Hail Mary.epub', parsed_title: 'Project Hail Mary', parsed_author: 'Andy Weir', reason: 'no_candidate_books' }, - { path: '/books/Andy Weir/The Martian.epub', parsed_title: 'The Martian', parsed_author: 'Andy Weir', reason: 'author_not_in_library' }, - ], + files_found: 3, + reconciled: 3, + unmatched: 0, + unmatched_units: 0, }) renderSettings() expect(await screen.findByText('settings.general.lastScan')).toBeInTheDocument() - expect(screen.getByText('settings.general.scanAllUnmatchedHint')).toBeInTheDocument() - expect(screen.queryByText('settings.general.scanUnmatchedAuthorUnknown')).not.toBeInTheDocument() + expect(screen.queryByRole('link', { name: /settings\.general\.reviewUnmatched/ })).not.toBeInTheDocument() }) it('soft-navigates to Root Folders from the Default library location link', async () => { diff --git a/web/src/pages/WantedPage.test.tsx b/web/src/pages/WantedPage.test.tsx index 3fd57f8d4..a6bbea40d 100644 --- a/web/src/pages/WantedPage.test.tsx +++ b/web/src/pages/WantedPage.test.tsx @@ -199,9 +199,9 @@ describe('WantedPage', () => { expect(await screen.findByText('No wanted books. Add an author to start tracking.')).toBeInTheDocument() const manualImport = screen.getByRole('link', { name: 'Import them' }) - expect(manualImport).toHaveAttribute('href', '/settings?tab=import') + expect(manualImport).toHaveAttribute('href', '/import?view=folder') const scanLibrary = screen.getByRole('link', { name: 'Scan Library' }) - expect(scanLibrary).toHaveAttribute('href', '/settings?tab=general') + expect(scanLibrary).toHaveAttribute('href', '/import') }) it('renders a row with the book title and its author', async () => { diff --git a/web/src/pages/import/AdoptionEditor.tsx b/web/src/pages/import/AdoptionEditor.tsx new file mode 100644 index 000000000..66c02e95d --- /dev/null +++ b/web/src/pages/import/AdoptionEditor.tsx @@ -0,0 +1,153 @@ +import { useEffect, useId, useRef, useState, type KeyboardEvent } from 'react' +import { useTranslation } from 'react-i18next' +import type { AdoptionBookRef, AdoptionItem, AdoptTarget, Book } from '../../api/client' +import { btn, btnSize } from '../../components/buttons' +import BookPicker from '../../components/import/BookPicker' +import CatalogueAdder from '../../components/import/CatalogueAdder' +import { adoptionHint, scorePercent, unitDisplayName } from './adoptionHint' + +interface Props { + item: AdoptionItem + onAdopt: (target: AdoptTarget, preview: AdoptionBookRef | null) => void | Promise + onCancel: () => void + // Opens the file list, for the row's Show files action. + showFiles?: boolean +} + +function refFromBook(b: Book): AdoptionBookRef { + return { + id: b.id, title: b.title, authorId: b.authorId, authorName: b.author?.authorName ?? '', + imageUrl: b.imageUrl, status: b.status, mediaType: b.mediaType ?? '', monitored: b.monitored, + } +} + +// AdoptionEditor opens in place under its row, never as a dialog: the +// suggestions as radio rows with their scores, a search of the library, +// prefilled and focused, and a collapsed metadata search that asks a provider +// only when submitted. Esc closes it and focus goes back to the row. +export default function AdoptionEditor({ item, onAdopt, onCancel, showFiles = false }: Props) { + const { t } = useTranslation() + const headingId = useId() + const searchRef = useRef(null) + const [chosen, setChosen] = useState(item.candidates[0]?.book ?? null) + const [searched, setSearched] = useState(null) + const name = unitDisplayName(item) + + useEffect(() => { searchRef.current?.focus({ preventScroll: showFiles }) }, [showFiles]) + + const onKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Escape') { + e.stopPropagation() + onCancel() + } + } + const pickSearched = (b: Book) => { + const ref = refFromBook(b) + setSearched(ref) + setChosen(ref) + } + const options: { book: AdoptionBookRef; score?: number }[] = [ + ...item.candidates, + ...(searched && !item.candidates.some(c => c.book.id === searched.id) ? [{ book: searched }] : []), + ] + const more = item.fileCount - item.members.length + + return ( +
+

+ {t('adoption.editor.heading', { name, defaultValue: 'Which book is {{name}}?' })} +

+

{adoptionHint(item, t).sentence}

+ + {options.length > 0 && ( +
+ + {t('adoption.editor.suggestions', 'Suggestions')} + +
+ {options.map(o => { + const pct = o.score === undefined ? null : scorePercent(o.score) + return ( + + ) + })} +
+
+ )} + +
+
+

{t('adoption.editor.searchLibrary', 'Search your library')}

+ + { + await onAdopt({ + foreignBookId: b.foreignBookId, + foreignAuthorId: b.author?.foreignAuthorId ?? '', + authorName: b.author?.authorName ?? '', + }, null) + }} + /> +
+
+ {item.members.length > 1 && ( +
+ + {t('adoption.editor.files', { count: item.fileCount, defaultValue: '{{count}} files' })} + +
    + {item.members.map(m =>
  • {m}
    • )} + {more > 0 &&
  • {t('adoption.editor.moreFiles', { count: more, defaultValue: 'and {{count}} more' })}
    • } +
+
+ )} +

+ {t('adoption.editor.inPlace', 'Files stay where they are. Picking a book already in your library leaves its owner and monitoring as they are.')} +

+
+
+ +
+ + +
+
+ ) +} diff --git a/web/src/pages/import/AdoptionFacets.tsx b/web/src/pages/import/AdoptionFacets.tsx new file mode 100644 index 000000000..4f9c48e60 --- /dev/null +++ b/web/src/pages/import/AdoptionFacets.tsx @@ -0,0 +1,108 @@ +import { forwardRef } from 'react' +import { useTranslation } from 'react-i18next' +import type { AdoptionFacets as Facets, AdoptionState, AdoptionSummary } from '../../api/client' +import FilterPopover, { FilterGroup } from '../../components/FilterPopover' +import type { AdoptionFilters } from './useAdoptionList' + +interface Props { + filters: AdoptionFilters + facets: Facets | null + summary: AdoptionSummary | null + onChange: (patch: Partial) => void +} + +const STATES: AdoptionState[] = ['pending', 'ignored', 'adopted'] + +// The toolbar over the list: search, which decisions to look at, and the +// reason and format facets behind one disclosure. An applied filter is always +// visible as a removable chip, the same rule BooksPage follows. +const AdoptionFacets = forwardRef(function AdoptionFacets({ filters, facets, summary, onChange }, searchRef) { + const { t } = useTranslation() + + const stateLabel = (s: AdoptionState) => { + const count = s === 'pending' ? summary?.pending : s === 'ignored' ? summary?.ignored : summary?.adopted + const label = t(`adoption.state.${s}`, s === 'pending' ? 'Needs a decision' : s === 'ignored' ? 'Ignored' : 'Adopted') + return count ? `${label} (${count})` : label + } + const reasonLabel = (code: string) => t(`adoption.reason.${code}`, code) + const formatLabel = (f: string) => (f === 'audiobook' ? t('common.audiobook', 'Audiobook') : t('common.ebook', 'Ebook')) + const pill = (active: boolean) => + `px-3 py-1 rounded-md text-xs font-medium transition-colors ${active ? 'bg-slate-300 dark:bg-zinc-700 text-slate-900 dark:text-white' : 'text-slate-600 dark:text-zinc-400 hover:text-slate-900 dark:hover:text-white'}` + const chip = 'inline-flex items-center gap-1 px-2 py-1 rounded-full bg-slate-200 dark:bg-zinc-800 text-xs text-slate-700 dark:text-zinc-300 hover:bg-slate-300 dark:hover:bg-zinc-700' + + return ( +
+
+ onChange({ search: e.target.value })} + placeholder={t('adoption.searchPlaceholder', 'Search titles, authors and folders')} + aria-label={t('adoption.searchLabel', 'Search books to decide')} + aria-keyshortcuts="/" + className="flex-1 bg-slate-200 dark:bg-zinc-800 border border-slate-300 dark:border-zinc-700 rounded px-3 py-2 text-sm focus:outline-none focus:border-slate-400 dark:focus:border-zinc-600 placeholder-slate-400 dark:placeholder-zinc-600" + /> +
+ {STATES.map(s => ( + + ))} +
+
+
+ + onChange({ reason })} + options={[ + { value: '', label: t('common.all', 'All') }, + ...(facets?.reasons ?? []).map(r => ({ value: r.value, label: `${reasonLabel(r.value)} (${r.count})` })), + ]} + /> + onChange({ format })} + options={[ + { value: '', label: t('common.all', 'All') }, + ...(facets?.formats ?? []).map(f => ({ value: f.value, label: `${formatLabel(f.value)} (${f.count})` })), + ]} + /> + + {filters.authorFolder && ( + + )} + {filters.reason && ( + + )} + {filters.format && ( + + )} +
+
+ ) +}) + +export default AdoptionFacets diff --git a/web/src/pages/import/AdoptionGroupRow.tsx b/web/src/pages/import/AdoptionGroupRow.tsx new file mode 100644 index 000000000..e367b3eb9 --- /dev/null +++ b/web/src/pages/import/AdoptionGroupRow.tsx @@ -0,0 +1,73 @@ +import { forwardRef, type KeyboardEvent } from 'react' +import { useTranslation } from 'react-i18next' +import { btn, btnSize } from '../../components/buttons' +import MoreMenu from '../../components/MoreMenu' +import type { AdoptionGroup } from './adoptionGroups' +import { actionCellCls, cellCls, rowCls } from './adoptionStyles' + +interface Props { + group: AdoptionGroup + open: boolean + focusable: boolean + onFocusRow: () => void + onKeyDown: (e: KeyboardEvent) => void + onToggle: () => void + onAddAuthor: (name: string) => void + onIgnoreFolder: (folder: string) => void +} + +// AdoptionGroupRow stands for every book on the page under one author folder +// whose author is not in the library. It says so once, offers Add author once, +// and discloses the books, which keep their own Choose book and Ignore. +const AdoptionGroupRow = forwardRef(function AdoptionGroupRow( + { group, open, focusable, onFocusRow, onKeyDown, onToggle, onAddAuthor, onIgnoreFolder }, ref, +) { + const { t } = useTranslation() + const panelLabel = open ? t('adoption.group.hide', 'Hide books') : t('adoption.group.show', 'Show books') + return ( + { if (e.target === e.currentTarget) onFocusRow() }} + className={rowCls(false, false)} + > + + + + +

+ {t('adoption.group.sentence', 'Author not in your library yet')} +

+ + +
+ + onIgnoreFolder(group.folder) }, + ]} + /> +
+ + + ) +}) + +export default AdoptionGroupRow diff --git a/web/src/pages/import/AdoptionRail.tsx b/web/src/pages/import/AdoptionRail.tsx new file mode 100644 index 000000000..7da804767 --- /dev/null +++ b/web/src/pages/import/AdoptionRail.tsx @@ -0,0 +1,60 @@ +import { useTranslation } from 'react-i18next' +import type { AdoptionFolderFacet } from '../../api/client' + +interface Props { + folders: AdoptionFolderFacet[] + total: number + activeFolder: string + onShow: (folder: string) => void +} + +// The folder rail is navigation: each author folder with the most undecided +// books is one item that filters the list, its count right aligned. An amber +// dot marks a folder whose author is not in the library, which the list then +// shows as one group with Add author. Actions live in the list, not here. +// +// A column beside the list on wide screens, a row of scrolling chips on narrow +// ones. +export default function AdoptionRail({ folders, total, activeFolder, onShow }: Props) { + const { t } = useTranslation() + if (folders.length === 0) return null + + const item = (key: string, label: string, count: number, dot: 'amber' | 'slate' | null, active: boolean, onClick: () => void, title?: string) => ( +
  • + +
    • + ) + + return ( + + ) +} diff --git a/web/src/pages/import/AdoptionRow.tsx b/web/src/pages/import/AdoptionRow.tsx new file mode 100644 index 000000000..96ea3dd5d --- /dev/null +++ b/web/src/pages/import/AdoptionRow.tsx @@ -0,0 +1,162 @@ +import { forwardRef, type KeyboardEvent } from 'react' +import { Link } from 'react-router' +import { useTranslation } from 'react-i18next' +import type { AdoptionItem } from '../../api/client' +import { btn, btnSize } from '../../components/buttons' +import MediaBadge from '../../components/MediaBadge' +import MoreMenu, { type MoreMenuItem } from '../../components/MoreMenu' +import { formatBytes } from '../../util/format' +import { adoptionHint, unitDisplayName } from './adoptionHint' +import { matchStrength, shortHint } from './adoptionMatch' +import type { Outcome } from './adoptionReducer' +import { rowCls, cellCls, actionCellCls } from './adoptionStyles' + +interface Props { + item: AdoptionItem + outcome: Outcome | undefined + error: string | undefined + note: 'keptBook' | undefined + expanded: boolean + focusable: boolean + inGroup: boolean + editorId: string + onFocusRow: () => void + onKeyDown: (e: KeyboardEvent) => void + onOpen: (showFiles: boolean) => void + onConfirm: () => void + onIgnore: () => void + onUndo: () => void + onAddAuthor: (name: string) => void +} + +// AdoptionRow is one book: what it is, one line on why it is here, and a fixed +// action cell holding the single most likely action plus a More menu. Only a +// strong suggestion (adoptionMatch) earns a one click Confirm; a possible one +// is a quiet link that opens the editor with it preselected. A decision turns +// the middle cell into a quiet line and the action into Undo, until the next +// fetch. +const AdoptionRow = forwardRef(function AdoptionRow( + { item, outcome, error, note, expanded, focusable, inGroup, editorId, onFocusRow, onKeyDown, onOpen, onConfirm, onIgnore, onUndo, onAddAuthor }, ref, +) { + const { t } = useTranslation() + const name = unitDisplayName(item) + const top = item.candidates[0] + const strength = matchStrength(item) + const full = adoptionHint(item, t) + const tooltip = [full.sentence, full.tooltip].filter(Boolean).join('\n') + const size = formatBytes(item.sizeBytes) + const pending = item.state === 'pending' && !outcome + const authorFirst = pending && !inGroup && !top && item.reason === 'author_not_in_library' && item.parsedAuthor !== '' + + const outcomeLine = (() => { + switch (outcome?.kind) { + case undefined: return null + case 'adopting': return t('adoption.outcome.adopting', { title: outcome.preview?.title ?? name, defaultValue: 'Adopting as {{title}}…' }) + case 'adopted': { + const book = outcome.item.book + const base = t('adoption.outcome.adopted', { title: book?.title ?? name, author: book?.authorName ?? '', defaultValue: 'Adopted as {{title}} by {{author}}.' }) + return outcome.item.bookCreated ? `${base} ${t('adoption.outcome.created', 'Added to your library, unmonitored.')}` : base + } + case 'ignoring': + case 'ignored': return t('adoption.outcome.ignored', 'Ignored. Later scans keep it out of this list.') + case 'undoing': return t('adoption.outcome.undoing', 'Undoing…') + case 'restored': return t('adoption.outcome.restored', 'Back in Needs a decision.') + } + })() + const canUndo = outcome?.kind === 'adopted' || outcome?.kind === 'ignored' + + const menu: MoreMenuItem[] = [] + if (pending) { + if (strength === 'strong' || authorFirst) menu.push({ label: t('adoption.choose', 'Choose book'), onSelect: () => onOpen(false) }) + if (item.fileCount > 1) menu.push({ label: t('adoption.showFiles', 'Show files'), onSelect: () => onOpen(true) }) + menu.push({ label: t('adoption.ignore', 'Ignore'), onSelect: onIgnore }) + } + + let primary: React.ReactNode + if (canUndo) { + primary = + } else if (outcome) { + primary = null + } else if (item.state === 'ignored' || item.state === 'adopted') { + primary = ( + + ) + } else if (strength === 'strong') { + primary = + } else if (authorFirst) { + primary = + } else { + primary = ( + + ) + } + + return ( + { if (e.target === e.currentTarget) onFocusRow() }} + className={rowCls(expanded, inGroup)} + > + +
    + {name} + +
    +

    + + {t('adoption.row.files', { count: item.fileCount, defaultValue: '{{count}} files' })}{size ? ` · ${size}` : ''} + + · {item.relPath} +

    + + + {outcomeLine ? ( +

    {outcomeLine}

    + ) : item.state === 'adopted' && item.book ? ( +

    + {t('adoption.row.adoptedAs', 'Adopted as')}{' '} + {item.book.title} +

    + ) : top ? ( +

    + + {strength === 'strong' ? t('adoption.match.strong', 'Strong match') : t('adoption.match.possible', 'Possible match')} + + {strength === 'strong' ? ( + {top.book.title} + ) : ( + + )} +

    + ) : inGroup ? ( + // The group row above already says why; repeating it per book is noise. + {shortHint(item, t)} + ) : ( +

    {shortHint(item, t)}

    + )} + {note === 'keptBook' && ( +

    + {t('adoption.outcome.keptBook', 'Files removed. The book stayed because it is now in use.')} +

    + )} + {error &&

    {error}

    } + + +
    + {primary} + {menu.length > 0 && ( + + )} +
    + + + ) +}) + +export default AdoptionRow diff --git a/web/src/pages/import/AdoptionSummary.tsx b/web/src/pages/import/AdoptionSummary.tsx new file mode 100644 index 000000000..80b9bc3eb --- /dev/null +++ b/web/src/pages/import/AdoptionSummary.tsx @@ -0,0 +1,87 @@ +import { useTranslation } from 'react-i18next' +import type { AdoptionScanStatus, AdoptionSummary as Summary } from '../../api/client' +import { btn, btnSize } from '../../components/buttons' +import { relativeTime } from './adoptionHint' + +interface Props { + summary: Summary | null + scan: AdoptionScanStatus | null + onScan: () => void + // Set after an author is added: the next useful step is a scan. + nudgeScan: boolean +} + +// The strip above the list. One number matters, the books waiting for a +// decision; files, the last scan, ignored and adopted are quieter stats beside +// it. Scan now is the one control. +export default function AdoptionSummary({ summary, scan, onScan, nudgeScan }: Props) { + const { t, i18n } = useTranslation() + const running = Boolean(scan?.running) + const pending = summary?.pending ?? 0 + const recent = scan?.ranAt ? Date.now() - Date.parse(scan.ranAt) < 60_000 : false + const ranAt = !scan?.ran ? '' : recent ? t('adoption.summary.justNow', 'just now') : relativeTime(scan.ranAt, i18n.language) + + const stat = (key: string, value: React.ReactNode, label: string, labelFirst = false) => ( +
  • + {value} + {label} +
    • + ) + + return ( +
    +
    +

    + {running ? ( + +

    +
      + {summary && summary.pendingFiles > 0 && stat('files', summary.pendingFiles, t('adoption.summary.files', { count: summary.pendingFiles, defaultValue: 'files' }))} + {stat('scan', ranAt || t('adoption.summary.never', 'never'), t('adoption.summary.lastScanLabel', 'last scan'), true)} + {summary && summary.adopted > 0 && stat('adopted', summary.adopted, t('adoption.summary.adoptedLabel', 'adopted'))} + {summary && summary.ignored > 0 && stat('ignored', summary.ignored, t('adoption.summary.ignoredLabel', 'ignored'))} +
    +
    + {nudgeScan && !running && ( +

    {t('adoption.summary.nudge', 'Author added. Scan now to match their files.')}

    + )} + +
    +
    + {scan?.error && ( +

    + {t('adoption.summary.scanError', { error: scan.error, defaultValue: 'The last scan did not finish: {{error}}' })} +

    + )} + {!scan?.error && scan?.ran && scan.noFilesFound && ( +

    + {t('adoption.summary.noFiles', 'The last scan found no book files. Check that the library folder is mounted, then scan again. Your decisions were kept.')} +

    + )} + {scan?.truncated && ( +

    + {t('adoption.summary.truncated', 'This library is larger than one scan lists. Adopt or ignore some books and scan again to see the rest.')} +

    + )} +
    + ) +} diff --git a/web/src/pages/import/AdoptionTable.tsx b/web/src/pages/import/AdoptionTable.tsx new file mode 100644 index 000000000..2cd0023df --- /dev/null +++ b/web/src/pages/import/AdoptionTable.tsx @@ -0,0 +1,165 @@ +import { Fragment, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react' +import { useTranslation } from 'react-i18next' +import type { AdoptionItem, AdoptionSort } from '../../api/client' +import SortHeader from '../../components/SortHeader' +import AdoptionEditor from './AdoptionEditor' +import AdoptionGroupRow from './AdoptionGroupRow' +import AdoptionRow from './AdoptionRow' +import { groupEntries, visibleEntries, type AdoptionListEntry } from './adoptionGroups' +import { matchStrength } from './adoptionMatch' +import type { AdoptionList } from './useAdoptionList' + +interface Props { + list: AdoptionList + onSearchShortcut: () => void + onAddAuthor: (name: string) => void + onIgnoreFolder: (folder: string) => void +} + +// Natural direction per column: scores, file counts and sizes read biggest +// first, names read A to Z. +const NATURAL_DESC: Record = { score: true, files: true, size: true, seen: true, title: false, folder: false } + +const entryKey = (e: AdoptionListEntry) => (e.kind === 'group' ? e.key : `item:${e.item.id}`) + +// AdoptionTable is the list: a real table on wide screens, stacked cards on +// narrow ones. Books sharing a missing author are one group row. One row holds +// the tab stop (roving tabindex); arrows move it, Enter opens a book's editor +// in place or a group's books, Esc closes the editor and returns focus to its +// row, i ignores, u undoes, and / jumps to the search box. +export default function AdoptionTable({ list, onSearchShortcut, onAddAuthor, onIgnoreFolder }: Props) { + const { t } = useTranslation() + const { state, filters } = list + const [focusIndex, setFocusIndex] = useState(0) + const [openGroups, setOpenGroups] = useState>(() => new Set()) + const [filesOpen, setFilesOpen] = useState(false) + const rowRefs = useRef<(HTMLTableRowElement | null)[]>([]) + const restoreFocusTo = useRef(null) + const entries = useMemo(() => visibleEntries(groupEntries(state.items), openGroups), [state.items, openGroups]) + const direction: 'asc' | 'desc' = filters.dir || (NATURAL_DESC[filters.sort] ? 'desc' : 'asc') + + useEffect(() => { setFocusIndex(i => Math.min(i, Math.max(0, entries.length - 1))) }, [entries.length]) + + // After the editor closes, focus goes back to the row it belonged to. + useEffect(() => { + if (state.expandedId === null && restoreFocusTo.current !== null) { + const index = entries.findIndex(e => entryKey(e) === restoreFocusTo.current) + rowRefs.current[index]?.focus() + restoreFocusTo.current = null + } + }, [state.expandedId, entries]) + + const sortBy = (key: AdoptionSort) => { + if (key === filters.sort) list.setFilter({ dir: direction === 'asc' ? 'desc' : 'asc' }) + else list.setFilter({ sort: key, dir: '' }) + } + const toggleGroup = (key: string) => setOpenGroups(prev => { + const next = new Set(prev) + if (next.has(key)) next.delete(key) + else next.add(key) + return next + }) + const openEditor = (item: AdoptionItem, showFiles: boolean) => { + setFilesOpen(showFiles) + list.expand(item.id) + } + const closeEditor = (item: AdoptionItem) => { + restoreFocusTo.current = `item:${item.id}` + list.collapse() + } + const confirm = (item: AdoptionItem) => { + const top = item.candidates[0] + if (top && matchStrength(item) === 'strong') void list.adopt(item, { bookId: top.book.id }, top.book) + } + + const onRowKey = (e: KeyboardEvent, index: number, entry: AdoptionListEntry) => { + if (e.target !== e.currentTarget) return + const move = (to: number) => { + e.preventDefault() + const next = Math.max(0, Math.min(entries.length - 1, to)) + setFocusIndex(next) + rowRefs.current[next]?.focus() + } + if (e.key === 'ArrowDown') return move(index + 1) + if (e.key === 'ArrowUp') return move(index - 1) + if (e.key === 'Home') return move(0) + if (e.key === 'End') return move(entries.length - 1) + if (e.key === '/') { e.preventDefault(); onSearchShortcut(); return } + if (entry.kind === 'group') { + if (e.key === 'Enter') { e.preventDefault(); toggleGroup(entry.key) } + return + } + const { item } = entry + const outcome = state.outcomes[item.id] + if (e.key === 'Enter') { + e.preventDefault() + if (outcome || item.state !== 'pending') return + if (state.expandedId === item.id) closeEditor(item) + else openEditor(item, false) + } else if (e.key === 'Escape' && state.expandedId === item.id) { + closeEditor(item) + } else if (e.key === 'i' && !outcome && item.state === 'pending') { + e.preventDefault() + void list.ignore(item) + } else if (e.key === 'u' && (outcome?.kind === 'adopted' || outcome?.kind === 'ignored' || (!outcome && item.state !== 'pending'))) { + e.preventDefault() + void list.undo(item) + } + } + + const captionKey = `adoption.caption.${filters.state}` + const captionDefault = filters.state === 'ignored' ? 'Ignored books in your library' : filters.state === 'adopted' ? 'Books you adopted from your library' : 'Books in your library that need a decision' + + return ( +
    + + + + + + + + + + + {entries.map((entry, index) => { + const common = { + ref: (el: HTMLTableRowElement | null) => { rowRefs.current[index] = el }, + focusable: index === focusIndex, + onFocusRow: () => setFocusIndex(index), + onKeyDown: (e: KeyboardEvent) => onRowKey(e, index, entry), + } + if (entry.kind === 'group') { + return ( + toggleGroup(entry.key)} onAddAuthor={onAddAuthor} onIgnoreFolder={onIgnoreFolder} /> + ) + } + const { item } = entry + const expanded = state.expandedId === item.id + const editorId = `adoption-editor-${item.id}` + return ( + + (expanded && !showFiles ? closeEditor(item) : openEditor(item, showFiles))} + onConfirm={() => confirm(item)} onIgnore={() => void list.ignore(item)} onUndo={() => void list.undo(item)} + onAddAuthor={onAddAuthor} /> + {expanded && ( + + + + )} + + ) + })} + +
    + {t(captionKey, captionDefault)}. {t('adoption.caption.keys', 'Arrow keys move between books, Enter opens one, i ignores, u undoes.')} +
    {t('adoption.col.actions', 'Actions')}
    + list.adopt(item, target, preview)} onCancel={() => closeEditor(item)} /> +
    +
    + ) +} diff --git a/web/src/pages/import/AdoptionView.test.tsx b/web/src/pages/import/AdoptionView.test.tsx new file mode 100644 index 000000000..ef80fa050 --- /dev/null +++ b/web/src/pages/import/AdoptionView.test.tsx @@ -0,0 +1,283 @@ +import { fireEvent, render, screen, waitFor, within } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { http, HttpResponse } from 'msw' +import { MemoryRouter } from 'react-router' +import { apiUrl, server } from '../../test/msw' +import type { AdoptionItem, AdoptionListResponse } from '../../api/client' +import AdoptionView from './AdoptionView' + +// AdoptionView talks to the server through the real api client, mocked at the +// network with MSW (T1), so a fetch spy sees exactly what the page requests. +// Strings render from en.json, so the assertions read like the page does. +vi.mock('react-i18next', async () => { + const en = (await import('../../i18n/locales/en.json')).default as Record + const lookup = (key: string): unknown => + key.split('.').reduce((n, p) => (n && typeof n === 'object' ? (n as Record)[p] : undefined), en) + const t = (key: string, options?: string | Record) => { + const opts = typeof options === 'object' ? options : {} + let value = lookup(key) + if (typeof opts.count === 'number') value = lookup(`${key}_${opts.count === 1 ? 'one' : 'other'}`) ?? value + if (typeof value !== 'string') value = typeof options === 'string' ? options : (opts.defaultValue as string) ?? key + return (value as string).replace(/\{\{(\w+)\}\}/g, (_, k) => String(opts[k] ?? '')) + } + return { useTranslation: () => ({ t, i18n: { language: 'en' } }) } +}) + +function item(overrides: Partial & Pick): AdoptionItem { + return { + kind: 'file', format: 'ebook', fileCount: 1, sizeBytes: 2048, relPath: `Andy Weir/Book ${overrides.id}.epub`, + rootPath: '/books', authorFolder: 'Andy Weir', parsedTitle: `Book ${overrides.id}`, parsedAuthor: 'Andy Weir', + reason: 'no_title_match', candidates: [], topScore: 0, state: 'pending', bookCreated: false, authorCreated: false, + members: [`Book ${overrides.id}.epub`], firstSeenAt: '2026-09-17T00:00:00Z', + ...overrides, + } +} + +const martian = { id: 42, title: 'The Martian', authorId: 7, authorName: 'Andy Weir', status: 'wanted', mediaType: 'ebook', monitored: true } + +function listResponse(items: AdoptionItem[], overrides: Partial = {}): AdoptionListResponse { + return { + items, + total: items.length, + facets: { reasons: [{ value: 'no_title_match', count: items.length }], formats: [{ value: 'ebook', count: items.length }], folders: [] }, + summary: { pending: items.length, pendingFiles: items.length, ignored: 0, adopted: 0 }, + scan: { ran: true, ranAt: new Date(Date.now() - 2 * 3600_000).toISOString(), running: false, filesFound: 10, truncated: false, noFilesFound: false }, + ...overrides, + } +} + +const suggested = item({ + id: 1, parsedTitle: 'The Martian', relPath: 'Andy Weir/The Martian.epub', + candidates: [{ book: martian, score: 0.97 }], topScore: 0.97, +}) +const weak = item({ + id: 4, parsedTitle: 'A Martyrs Tale', relPath: 'Andy Weir/A Martyrs Tale.epub', + candidates: [{ book: martian, score: 0.82 }], topScore: 0.82, +}) +const unsuggested = item({ id: 2, parsedTitle: 'Mystery Notes', relPath: 'Andy Weir/Mystery Notes.epub' }) + +let fetchSpy: { mock: { calls: Parameters[] }; mockRestore: () => void } + +function requestedURLs(): string[] { + return fetchSpy.mock.calls.map(([input]: Parameters) => (typeof input === 'string' ? input : input instanceof URL ? input.href : (input as Request).url)) +} + +function serve(response: AdoptionListResponse) { + server.use( + http.get(apiUrl('/library/unmatched'), () => HttpResponse.json(response)), + http.get(apiUrl('/library/unmatched/summary'), () => HttpResponse.json({ ...response.summary, scan: response.scan })), + ) +} + +async function renderView() { + render() + return screen.findByRole('table', { name: /Books in your library that need a decision/ }) +} + +beforeEach(() => { + localStorage.clear() + fetchSpy = vi.spyOn(window, 'fetch') +}) + +afterEach(() => { + fetchSpy.mockRestore() +}) + +describe('AdoptionView', () => { + it('confirms a strong suggestion in one click, then undoes it', async () => { + serve(listResponse([suggested, unsuggested])) + let adoptBody: unknown = null + server.use( + http.post(apiUrl('/library/unmatched/1/adopt'), async ({ request }) => { + adoptBody = await request.json() + return HttpResponse.json({ ...suggested, state: 'adopted', book: martian }) + }), + http.post(apiUrl('/library/unmatched/1/undo'), () => HttpResponse.json(suggested)), + ) + const table = await renderView() + + const row = within(table).getByRole('row', { name: 'The Martian' }) + expect(within(row).getByText('Strong match')).toBeInTheDocument() + fireEvent.click(within(row).getByRole('button', { name: 'Confirm' })) + + expect(await within(row).findByText(/Adopted as The Martian by Andy Weir\./)).toBeInTheDocument() + expect(adoptBody).toEqual({ bookId: 42 }) + + fireEvent.click(within(row).getByRole('button', { name: 'Undo' })) + expect(await within(row).findByRole('button', { name: 'Confirm' })).toBeInTheDocument() + }) + + it('says quietly when Undo kept a book that is now in use', async () => { + serve(listResponse([suggested])) + server.use( + http.post(apiUrl('/library/unmatched/1/adopt'), () => HttpResponse.json({ ...suggested, state: 'adopted', book: martian, bookCreated: true })), + http.post(apiUrl('/library/unmatched/1/undo'), () => + HttpResponse.json({ ...suggested, message: 'The files are no longer adopted. The book it added stays in your library because it has been used since.' })), + ) + const table = await renderView() + const row = within(table).getByRole('row', { name: 'The Martian' }) + fireEvent.click(within(row).getByRole('button', { name: 'Confirm' })) + fireEvent.click(await within(row).findByRole('button', { name: 'Undo' })) + + expect(await within(row).findByText('Files removed. The book stayed because it is now in use.')).toBeInTheDocument() + expect(within(row).getByRole('button', { name: 'Confirm' })).toBeInTheDocument() + expect(within(row).queryByRole('alert')).toBeNull() + }) + + it('says nothing extra when Undo removed the book too', async () => { + serve(listResponse([suggested])) + server.use( + http.post(apiUrl('/library/unmatched/1/adopt'), () => HttpResponse.json({ ...suggested, state: 'adopted', book: martian })), + http.post(apiUrl('/library/unmatched/1/undo'), () => HttpResponse.json(suggested)), + ) + const table = await renderView() + const row = within(table).getByRole('row', { name: 'The Martian' }) + fireEvent.click(within(row).getByRole('button', { name: 'Confirm' })) + fireEvent.click(await within(row).findByRole('button', { name: 'Undo' })) + await within(row).findByRole('button', { name: 'Confirm' }) + expect(within(row).queryByText(/The book stayed/)).toBeNull() + }) + + it('reverts an optimistic adopt and shows the error on the row', async () => { + serve(listResponse([suggested])) + server.use( + http.post(apiUrl('/library/unmatched/1/adopt'), () => + HttpResponse.json({ error: 'Provenance.epub already belongs to a book in your library.' }, { status: 409 })), + ) + const table = await renderView() + const row = within(table).getByRole('row', { name: 'The Martian' }) + fireEvent.click(within(row).getByRole('button', { name: 'Confirm' })) + + expect(await within(row).findByRole('alert')).toHaveTextContent('already belongs to a book') + expect(within(row).getByRole('button', { name: 'Confirm' })).toBeInTheDocument() + }) + + it('offers a weak suggestion as a possible match that opens the editor preselected', async () => { + serve(listResponse([weak])) + server.use(http.get(apiUrl('/book'), () => HttpResponse.json({ items: [], total: 0 }))) + const table = await renderView() + const row = within(table).getByRole('row', { name: 'A Martyrs Tale' }) + + expect(within(row).getByText('Possible match')).toBeInTheDocument() + expect(within(row).queryByRole('button', { name: 'Confirm' })).toBeNull() + expect(within(row).getByRole('button', { name: 'Choose book' })).toBeInTheDocument() + + fireEvent.click(within(row).getByRole('button', { name: 'The Martian' })) + const editor = await screen.findByRole('region', { name: 'Which book is A Martyrs Tale?' }) + expect(within(editor).getByRole('radio', { name: /The Martian/ })).toBeChecked() + expect(within(editor).getByRole('button', { name: 'Adopt as The Martian' })).toBeEnabled() + }) + + it('asks a metadata provider only when the search is submitted', async () => { + serve(listResponse([unsuggested])) + server.use(http.get(apiUrl('/search/book'), () => HttpResponse.json([]))) + const table = await renderView() + + const providerCalls = () => requestedURLs().filter(u => u.includes('/search/book') || u.includes('/book/lookup')) + fireEvent.click(within(table).getByRole('button', { name: 'Choose book' })) + await screen.findByRole('region', { name: 'Which book is Mystery Notes?' }) + fireEvent.click(screen.getByRole('button', { name: 'Not in your library? Search metadata' })) + fireEvent.change(screen.getByRole('textbox', { name: 'Search metadata' }), { target: { value: 'mystery notes weir' } }) + expect(providerCalls()).toEqual([]) + + fireEvent.click(screen.getByRole('button', { name: 'Search' })) + await waitFor(() => expect(providerCalls()).toHaveLength(1)) + expect(providerCalls()[0]).toContain('/search/book?term=mystery%20notes%20weir') + }) + + it('is operable from the keyboard: arrows, Enter, Esc and i', async () => { + serve(listResponse([suggested, unsuggested])) + server.use(http.get(apiUrl('/book'), () => HttpResponse.json({ items: [], total: 0 }))) + let ignored = false + server.use(http.post(apiUrl('/library/unmatched/2/ignore'), () => { + ignored = true + return HttpResponse.json({ ...unsuggested, state: 'ignored' }) + })) + const table = await renderView() + const [first, second] = within(table).getAllByRole('row').slice(1) + + expect(first).toHaveAttribute('tabindex', '0') + expect(second).toHaveAttribute('tabindex', '-1') + first.focus() + fireEvent.keyDown(first, { key: 'ArrowDown' }) + expect(second).toHaveFocus() + expect(second).toHaveAttribute('tabindex', '0') + + fireEvent.keyDown(second, { key: 'Enter' }) + const editor = await screen.findByRole('region', { name: 'Which book is Mystery Notes?' }) + expect(within(editor).getByRole('textbox', { name: 'Search your library' })).toHaveFocus() + + fireEvent.keyDown(within(editor).getByRole('textbox', { name: 'Search your library' }), { key: 'Escape' }) + await waitFor(() => expect(screen.queryByRole('region', { name: /Which book is/ })).toBeNull()) + expect(second).toHaveFocus() + + fireEvent.keyDown(second, { key: 'i' }) + expect(await within(second).findByText(/Ignored\. Later scans keep it out of this list\./)).toBeInTheDocument() + expect(ignored).toBe(true) + }) + + it('shows the never scanned state with a scan action', async () => { + serve(listResponse([], { scan: { ran: false, running: false, filesFound: 0, truncated: false, noFilesFound: false } })) + render() + expect(await screen.findByText('Your library has not been scanned yet')).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Scan library' })).toBeInTheDocument() + expect(screen.getByText('never')).toBeInTheDocument() + }) + + it('shows the all matched state after a scan with nothing left', async () => { + serve(listResponse([])) + render() + expect(await screen.findByText('Every book in your library is matched')).toBeInTheDocument() + expect(screen.getByText('books need a decision')).toBeInTheDocument() + }) + + it('says when the scan was truncated and when one is running', async () => { + serve(listResponse([unsuggested], { + scan: { ran: true, ranAt: new Date().toISOString(), running: true, filesFound: 60000, truncated: true, noFilesFound: false }, + })) + await renderView() + expect(screen.getByText('Scanning your library…')).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Scanning…' })).toBeDisabled() + expect(screen.getByText(/larger than one scan lists/)).toBeInTheDocument() + }) + + it('lists ignored books with Unignore and an empty ignored state', async () => { + serve(listResponse([])) + render() + await screen.findByText('Every book in your library is matched') + + const ignoredItem = item({ id: 3, parsedTitle: 'Old Manual', state: 'ignored' }) + server.use(http.get(apiUrl('/library/unmatched'), ({ request }) => + HttpResponse.json(new URL(request.url).searchParams.get('state') === 'ignored' ? listResponse([ignoredItem]) : listResponse([])))) + fireEvent.click(screen.getByRole('radio', { name: /Ignored/ })) + const table = await screen.findByRole('table', { name: /Ignored books in your library/ }) + expect(within(table).getByRole('button', { name: 'Unignore' })).toBeInTheDocument() + }) + + it('shows books whose author is missing as one group with a single Add author', async () => { + const becky = (id: number, title: string) => item({ + id, parsedTitle: title, parsedAuthor: 'Becky Chambers', authorFolder: 'Becky Chambers', + relPath: `Becky Chambers/${title}.epub`, reason: 'author_not_in_library', fileCount: 7, + }) + serve(listResponse([becky(11, 'Record of a Spaceborn Few'), unsuggested, becky(12, 'A Closed and Common Orbit')], { + facets: { reasons: [], formats: [], folders: [{ folder: 'Becky Chambers', units: 2, files: 14, notInLibrary: 2, author: 'Becky Chambers' }] }, + })) + const table = await renderView() + + const group = within(table).getByRole('row', { name: 'Becky Chambers, 2 books' }) + expect(within(group).getByText('2 books, 14 files')).toBeInTheDocument() + expect(within(table).getAllByRole('button', { name: 'Add author' })).toHaveLength(1) + expect(within(table).queryByRole('row', { name: 'Record of a Spaceborn Few' })).toBeNull() + + // The rail is navigation only: one item per folder, no actions. + const rail = screen.getByRole('navigation', { name: 'Folders with the most books to decide' }) + expect(within(rail).queryByRole('button', { name: 'Add author' })).toBeNull() + expect(within(rail).getByRole('button', { name: /Becky Chambers/ })).toHaveAttribute('title', 'Becky Chambers is not in your library') + + fireEvent.click(within(group).getByRole('button', { name: /14 files/ })) + const inner = await within(table).findByRole('row', { name: 'Record of a Spaceborn Few' }) + expect(within(inner).getByRole('button', { name: 'Choose book' })).toBeInTheDocument() + expect(within(inner).getByRole('button', { name: 'More actions for Record of a Spaceborn Few' })).toBeInTheDocument() + expect(within(inner).queryByRole('button', { name: 'Add author' })).toBeNull() + }) +}) diff --git a/web/src/pages/import/AdoptionView.tsx b/web/src/pages/import/AdoptionView.tsx new file mode 100644 index 000000000..1d85b38fc --- /dev/null +++ b/web/src/pages/import/AdoptionView.tsx @@ -0,0 +1,132 @@ +import { useEffect, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import AddToLibraryModal from '../../components/AddToLibraryModal' +import { btn, btnSize } from '../../components/buttons' +import Pagination from '../../components/Pagination' +import { useConfirmDialog } from '../../components/useConfirmDialog' +import AdoptionFacets from './AdoptionFacets' +import AdoptionRail from './AdoptionRail' +import AdoptionSummary from './AdoptionSummary' +import AdoptionTable from './AdoptionTable' +import { defaultAdoptionFilters, useAdoptionList } from './useAdoptionList' + +// AdoptionView is the Import page's "In your library" view. It lays out the +// summary strip, the folder rail, the toolbar and the list, and chooses which +// designed state to show when there is no list: never scanned, all matched, +// nothing ignored or adopted yet, or nothing matching the filters. The data +// and every decision live in useAdoptionList. +export default function AdoptionView() { + const { t } = useTranslation() + const list = useAdoptionList() + const { state, filters } = list + const { confirm, confirmDialog } = useConfirmDialog() + const searchRef = useRef(null) + const [addAuthor, setAddAuthor] = useState(null) + const [nudgeScan, setNudgeScan] = useState(false) + + // "/" jumps to the search from anywhere in the view but a text field. + useEffect(() => { + const onKey = (e: KeyboardEvent) => { + const el = e.target as HTMLElement | null + if (e.key !== '/' || e.defaultPrevented || el?.closest('input, textarea, select, [contenteditable="true"]')) return + e.preventDefault() + searchRef.current?.focus() + } + document.addEventListener('keydown', onKey) + return () => document.removeEventListener('keydown', onKey) + }, []) + + const scan = () => { + setNudgeScan(false) + void list.startScan() + } + + const ignoreFolder = async (folder: string) => { + if (!await confirm({ + title: t('adoption.ignoreFolder.title', { folder, defaultValue: 'Ignore every book in {{folder}}?' }), + body: t('adoption.ignoreFolder.body', 'They leave this list and later scans keep them out. You can unignore any of them from the Ignored list.'), + confirmLabel: t('adoption.ignoreFolder.confirm', 'Ignore folder'), + })) return + await list.ignoreFolder(folder) + } + + const folders = filters.state === 'pending' ? state.facets?.folders ?? [] : [] + const filtered = filters.search !== '' || filters.reason !== '' || filters.format !== '' || filters.authorFolder !== '' + const empty = state.loaded && !state.loading && state.items.length === 0 && !state.loadError + + const emptyState = () => { + const box = (title: string, body: string, action?: React.ReactNode) => ( +
    +

    {title}

    +

    {body}

    + {action &&
    {action}
    } +
    + ) + if (filtered) { + return box( + t('adoption.empty.filteredTitle', 'No books match these filters'), + t('adoption.empty.filteredBody', 'Try another search, or clear the filters to see every book.'), + , + ) + } + if (filters.state === 'ignored') return box(t('adoption.empty.ignoredTitle', 'Nothing is ignored'), t('adoption.empty.ignoredBody', 'Books you ignore land here, and you can bring any of them back.')) + if (filters.state === 'adopted') return box(t('adoption.empty.adoptedTitle', 'Nothing adopted yet'), t('adoption.empty.adoptedBody', 'Books you adopt stay here for 30 days, with Undo.')) + if (!state.scan?.ran) { + return box( + t('adoption.empty.neverTitle', 'Your library has not been scanned yet'), + t('adoption.empty.neverBody', 'A scan looks through your library folders and lists every book it cannot match to your library, so you can decide what each one is. Files are never moved.'), + , + ) + } + return box( + t('adoption.empty.doneTitle', 'Every book in your library is matched'), + t('adoption.empty.doneBody', 'Nothing needs a decision. New files show up here after the next scan.'), + ) + } + + return ( +
    + {confirmDialog} + + +
    0 ? 'lg:grid lg:grid-cols-[15rem_minmax(0,1fr)] lg:gap-6' : ''}> + list.setFilter({ authorFolder: folder })} + /> +
    + + + {state.loadError && ( +

    {state.loadError}

    + )} + {!state.loaded ? ( +
    {t('common.loading', 'Loading...')}
    + ) : empty ? ( + emptyState() + ) : ( +
    + searchRef.current?.focus()} onAddAuthor={setAddAuthor} onIgnoreFolder={folder => void ignoreFolder(folder)} /> +
    + )} + +
    +
    + + {addAuthor !== null && ( + setAddAuthor(null)} + onAdded={() => { setAddAuthor(null); setNudgeScan(true) }} + /> + )} +
    + ) +} diff --git a/web/src/pages/import/FolderImportRow.tsx b/web/src/pages/import/FolderImportRow.tsx new file mode 100644 index 000000000..89b446c16 --- /dev/null +++ b/web/src/pages/import/FolderImportRow.tsx @@ -0,0 +1,161 @@ +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { api } from '../../api/client' +import type { BatchImportResult, Book, ScanItem } from '../../api/client' +import { btn, btnSize } from '../../components/buttons' +import BookPicker from '../../components/import/BookPicker' +import CatalogueAdder from '../../components/import/CatalogueAdder' +import { groupHeading, type RowState } from './folderImport' + +function bookLabel(b: Book): string { + return b.author?.authorName ? `${b.title} · ${b.author.authorName}` : b.title +} + +interface ImportRowProps { + item: ScanItem + row: RowState | undefined + result: BatchImportResult | undefined + importing: boolean + onPick: (b: Book) => void + onToggle: () => void + onFormat: (f: string) => void + onImport: () => void +} + +export default function FolderImportRow({ item, row, result, importing, onPick, onToggle, onFormat, onImport }: ImportRowProps) { + const { t } = useTranslation() + const [overriding, setOverriding] = useState(false) + const chosen = row?.chosen ?? null + const accepted = result?.accepted + + const badgeCls: Record = { + confident: 'bg-emerald-100 dark:bg-emerald-950 text-emerald-700 dark:text-emerald-400', + ambiguous: 'bg-amber-100 dark:bg-amber-950 text-amber-700 dark:text-amber-400', + none: 'bg-slate-200 dark:bg-zinc-800 text-slate-600 dark:text-zinc-400', + } + + return ( +
    +
    + +
    +
    + {item.name} + + {t(`manualImport.group.${item.match}`, groupHeading(item.match))} + + + {item.detectedFormat} + +
    + {/* Full source path so the user can tell which file each row refers to; + the basename alone is ambiguous across folders (#1435). */} +
    {item.path}
    +
    + {t('manualImport.parsed', { title: item.parsedTitle || '?', author: item.parsedAuthor || '?', defaultValue: `parsed: ${item.parsedTitle || '?'} / ${item.parsedAuthor || '?'}` })} +
    + + {/* Resolution area */} +
    + {chosen && !overriding && ( +
    + → {bookLabel(chosen)} + {!accepted && ( + + )} +
    + )} + + {/* Ambiguous candidate picker */} + {item.match === 'ambiguous' && !chosen && item.candidates && item.candidates.length > 0 && ( +
    + {t('manualImport.pickCandidate', 'Pick the correct book')} + {item.candidates.map(c => ( + + ))} +
    + )} + + {/* None / override: search the existing catalogue, or add the book + from provider metadata when the library has nothing to bind to. */} + {((item.match === 'none' && !chosen) || overriding) && ( + <> + { onPick(b); setOverriding(false) }} + onCancel={overriding ? () => setOverriding(false) : undefined} + /> + {/* searchOnAdd is deliberately false: the file is already on + disk, so an indexer search would grab a second copy. */} + { + const created = await api.addBook({ + foreignBookId: b.foreignBookId, + // May be empty (a Google Books or DNB result carries a + // name but no author id); the backend resolves it by name. + foreignAuthorId: b.author?.foreignAuthorId ?? '', + authorName: b.author?.authorName ?? '', + searchOnAdd: false, + }) + onPick(created) + setOverriding(false) + }} + /> + + )} +
    +
    + +
    + + +
    +
    + + {result && ( +

    + {accepted + ? t('manualImport.queued', 'Queued — importing in the background; watch the Queue for progress.') + : t('manualImport.failed', { error: result.error || 'failed', defaultValue: `Failed: ${result.error || 'failed'}` })} +

    + )} +
    + ) +} diff --git a/web/src/pages/ManualImportPage.test.tsx b/web/src/pages/import/FolderImportView.test.tsx similarity index 97% rename from web/src/pages/ManualImportPage.test.tsx rename to web/src/pages/import/FolderImportView.test.tsx index 050f7e30e..75dce2871 100644 --- a/web/src/pages/ManualImportPage.test.tsx +++ b/web/src/pages/import/FolderImportView.test.tsx @@ -17,7 +17,7 @@ vi.mock('react-i18next', () => ({ }), })) -vi.mock('../api/client', () => ({ +vi.mock('../../api/client', () => ({ api: { scanFolder: vi.fn(), batchImport: vi.fn(), @@ -30,9 +30,9 @@ vi.mock('../api/client', () => ({ }, })) -import { api } from '../api/client' -import type { Book, FolderScanResponse } from '../api/client' -import ManualImportPage from './ManualImportPage' +import { api } from '../../api/client' +import type { Book, FolderScanResponse } from '../../api/client' +import FolderImportView from './FolderImportView' const mockScan = api.scanFolder as ReturnType const mockBatch = api.batchImport as ReturnType @@ -68,7 +68,7 @@ function scanResult(): FolderScanResponse { async function scan() { mockScan.mockResolvedValue(scanResult()) - render() + render() fireEvent.change(screen.getByPlaceholderText('manualImport.pathPlaceholder'), { target: { value: '/dl' } }) fireEvent.click(screen.getByText('manualImport.scan')) await waitFor(() => expect(screen.getByText('Confident Book')).toBeInTheDocument()) @@ -79,7 +79,7 @@ function rowImportButtons() { return screen.getAllByText('manualImport.import') as HTMLButtonElement[] } -describe('ManualImportPage', () => { +describe('FolderImportView', () => { beforeEach(() => vi.clearAllMocks()) it('renders scan results grouped by match status', async () => { diff --git a/web/src/pages/import/FolderImportView.tsx b/web/src/pages/import/FolderImportView.tsx new file mode 100644 index 000000000..472284017 --- /dev/null +++ b/web/src/pages/import/FolderImportView.tsx @@ -0,0 +1,233 @@ +import { useMemo, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { api } from '../../api/client' +import type { BatchImportItem, BatchImportResult, Book, ScanItem } from '../../api/client' +import { btn, btnSize } from '../../components/buttons' +import FolderImportRow from './FolderImportRow' +import { groupHeading, type RowState } from './folderImport' + +// FolderImportView is the Import page's "From a folder" view, the manual +// import wizard (#1236). It scans a folder with +// the recursive bulk-import scan (#1434), groups the discovered units by match +// confidence, and lets the user resolve each one before importing: +// - confident: a preselected catalogue match, deselectable and overridable; +// - ambiguous: a candidate picker over the returned catalogue candidates; +// - none: a search box to bind the unit to an EXISTING catalogue book, plus +// a metadata search that creates the book (and its author) when the library +// has no row to bind to (#1719). +// Resolved units are imported (per-unit or in bulk) through the batch endpoint, +// which reports per-item success/failure. + +const MATCH_ORDER: ScanItem['match'][] = ['confident', 'ambiguous', 'none'] + +export default function FolderImportView() { + const { t } = useTranslation() + const [path, setPath] = useState('') + const [scanning, setScanning] = useState(false) + const [items, setItems] = useState(null) + const [truncated, setTruncated] = useState(false) + const [rows, setRows] = useState>({}) + const [results, setResults] = useState>({}) + const [importingPaths, setImportingPaths] = useState>(() => new Set()) + const [scanError, setScanError] = useState('') + const [importError, setImportError] = useState('') + + const handleScan = async () => { + const p = path.trim() + if (!p) return + setScanning(true) + setScanError('') + setImportError('') + setItems(null) + setResults({}) + try { + const r = await api.scanFolder(p) + setItems(r.items) + setTruncated(r.truncated) + const init: Record = {} + for (const it of r.items) { + const chosen = it.match === 'confident' && it.book ? it.book : null + init[it.path] = { chosen, format: it.detectedFormat || '', selected: Boolean(chosen) } + } + setRows(init) + } catch (e) { + setScanError(e instanceof Error ? e.message : 'Scan failed') + } finally { + setScanning(false) + } + } + + const patchRow = (unitPath: string, patch: Partial) => + setRows(prev => ({ ...prev, [unitPath]: { ...prev[unitPath], ...patch } })) + + const pickBook = (unitPath: string, book: Book) => + patchRow(unitPath, { chosen: book, selected: true }) + + const toggleSelected = (unitPath: string) => + setRows(prev => { + const r = prev[unitPath] + if (!r || !r.chosen) return prev + return { ...prev, [unitPath]: { ...r, selected: !r.selected } } + }) + + // A path is submittable when it has a chosen book, is selected, and hasn't + // already been accepted by a prior import. A `none` unit the user never + // resolved has chosen=null, so it is never submitted. + const submittablePaths = useMemo(() => { + if (!items) return [] + return items + .map(it => it.path) + .filter(p => { + const r = rows[p] + return Boolean(r?.chosen) && r.selected && !results[p]?.accepted + }) + }, [items, rows, results]) + + const submit = async (paths: string[]) => { + const batch: BatchImportItem[] = [] + for (const p of paths) { + const r = rows[p] + if (!r?.chosen || !r.selected) continue + if (results[p]?.accepted) continue + batch.push({ path: p, bookId: r.chosen.id, format: r.format || undefined }) + } + if (batch.length === 0) return + setImportError('') + setImportingPaths(prev => { const n = new Set(prev); batch.forEach(b => n.add(b.path)); return n }) + try { + const res = await api.batchImport(batch) + setResults(prev => { + const n = { ...prev } + for (const r of res.results) n[r.path] = r + return n + }) + } catch (e) { + setImportError(e instanceof Error ? e.message : 'Import failed') + } finally { + setImportingPaths(prev => { const n = new Set(prev); batch.forEach(b => n.delete(b.path)); return n }) + } + } + + const allSelected = submittablePaths.length > 0 && + submittablePaths.every(p => rows[p]?.selected) + + const toggleSelectAll = () => { + if (!items) return + // Resolved, not-yet-imported units only — a `none` unit with no book can't + // be selected. + const resolvable = items + .map(it => it.path) + .filter(p => rows[p]?.chosen && !results[p]?.accepted) + const turnOn = !resolvable.every(p => rows[p]?.selected) + setRows(prev => { + const n = { ...prev } + for (const p of resolvable) n[p] = { ...n[p], selected: turnOn } + return n + }) + } + + const groups = useMemo(() => { + const g: Record = { confident: [], ambiguous: [], none: [] } + for (const it of items ?? []) g[it.match].push(it) + return g + }, [items]) + + return ( +
    +

    + {t('manualImport.description', 'Scan a folder of files already on disk, match each book to your library, and import them. A file with no match can be added from a metadata search.')} +

    + +
    + setPath(e.target.value)} + onKeyDown={e => { if (e.key === 'Enter') handleScan() }} + placeholder={t('manualImport.pathPlaceholder', '/downloads/books')} + aria-label={t('manualImport.pathLabel', 'Folder to scan')} + className="flex-1 px-3 py-2 rounded-md border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-sm" + /> + +
    + + {scanError && ( +

    {scanError}

    + )} + + {items && items.length === 0 && ( +

    + {t('manualImport.empty', 'No book files or folders found here.')} +

    + )} + + {items && items.length > 0 && ( + <> + {truncated && ( +

    + {t('manualImport.truncated', 'Showing the first 1000 units; narrow the folder to see the rest.')} +

    + )} + + {/* Select-all / bulk import bar */} +
    + + +
    + + {importError && ( +

    {importError}

    + )} + + {MATCH_ORDER.map(match => { + const group = groups[match] + if (group.length === 0) return null + return ( +
    +

    + {t(`manualImport.group.${match}`, groupHeading(match))} ({group.length}) +

    +
    + {group.map(it => ( + pickBook(it.path, b)} + onToggle={() => toggleSelected(it.path)} + onFormat={f => patchRow(it.path, { format: f })} + onImport={() => submit([it.path])} + /> + ))} +
    +
    + ) + })} + + )} +
    + ) +} + diff --git a/web/src/pages/import/ImportPage.test.tsx b/web/src/pages/import/ImportPage.test.tsx new file mode 100644 index 000000000..45e7b7a6b --- /dev/null +++ b/web/src/pages/import/ImportPage.test.tsx @@ -0,0 +1,51 @@ +import { fireEvent, render, screen } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' +import { MemoryRouter, useLocation } from 'react-router' +import ImportPage from './ImportPage' + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ t: (key: string, fallback?: unknown) => (typeof fallback === 'string' ? fallback : key) }), +})) +vi.mock('../../auth/AuthContext', () => ({ useAuth: () => ({ isAdmin: true }) })) +vi.mock('./AdoptionView', () => ({ default: () =>
    })) +vi.mock('./FolderImportView', () => ({ default: () =>
    })) + +function Location() { + const loc = useLocation() + return {loc.pathname + loc.search} +} + +function renderAt(url: string) { + render( + + + + , + ) +} + +describe('ImportPage', () => { + it('opens on "In your library" by default', () => { + renderAt('/import') + expect(screen.getByRole('tab', { name: 'In your library' })).toHaveAttribute('aria-selected', 'true') + expect(screen.getByTestId('adoption-view')).toBeInTheDocument() + }) + + it('addresses the folder view as ?view=folder, which ImportHints links to', () => { + renderAt('/import?view=folder') + expect(screen.getByRole('tab', { name: 'From a folder' })).toHaveAttribute('aria-selected', 'true') + expect(screen.getByRole('tabpanel')).toContainElement(screen.getByTestId('folder-view')) + }) + + it('switches views by click and by arrow key', () => { + renderAt('/import') + fireEvent.click(screen.getByRole('tab', { name: 'From a folder' })) + expect(screen.getByTestId('location')).toHaveTextContent('/import?view=folder') + + const folderTab = screen.getByRole('tab', { name: 'From a folder' }) + folderTab.focus() + fireEvent.keyDown(folderTab, { key: 'ArrowRight' }) + expect(screen.getByTestId('location')).toHaveTextContent(/^\/import$/) + expect(screen.getByRole('tab', { name: 'In your library' })).toHaveFocus() + }) +}) diff --git a/web/src/pages/import/ImportPage.tsx b/web/src/pages/import/ImportPage.tsx new file mode 100644 index 000000000..837c37e19 --- /dev/null +++ b/web/src/pages/import/ImportPage.tsx @@ -0,0 +1,97 @@ +import { useEffect, useRef, type KeyboardEvent } from 'react' +import { useSearchParams } from 'react-router' +import { useTranslation } from 'react-i18next' +import { useAuth } from '../../auth/AuthContext' +import AdoptionView from './AdoptionView' +import FolderImportView from './FolderImportView' + +type View = 'library' | 'folder' + +const VIEWS: View[] = ['library', 'folder'] + +// ImportPage is where files already on disk come into Bindery, two ways: +// - "In your library" (default): the books a library scan found but could +// not match, adopted in place (AdoptionView); +// - "From a folder" (?view=folder): pick a folder, match and import it, +// moving files into the library (FolderImportView). +// Both used to live apart (a read only table in Settings, a folder scan in +// Settings > Import and this page); this is now the one place. +export default function ImportPage() { + const { t } = useTranslation() + const { isAdmin } = useAuth() + const [params, setParams] = useSearchParams() + const view: View = params.get('view') === 'folder' ? 'folder' : 'library' + const tabRefs = useRef>({ library: null, folder: null }) + + useEffect(() => { + document.title = `${t('importPage.title', 'Import')} · Bindery` + return () => { document.title = 'Bindery' } + }, [t]) + + const select = (next: View) => { + const p = new URLSearchParams(params) + if (next === 'folder') p.set('view', 'folder') + else p.delete('view') + setParams(p, { replace: true }) + } + + // Tabs follow the WAI-ARIA pattern: arrows move between them. + const onTabKey = (e: KeyboardEvent) => { + if (e.key !== 'ArrowRight' && e.key !== 'ArrowLeft') return + e.preventDefault() + const next = VIEWS[(VIEWS.indexOf(view) + (e.key === 'ArrowRight' ? 1 : VIEWS.length - 1)) % VIEWS.length] + select(next) + tabRefs.current[next]?.focus() + } + + const tabCls = (active: boolean) => + `px-3 py-1.5 rounded-md text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-emerald-500 ${ + active + ? 'bg-white dark:bg-zinc-800 text-slate-900 dark:text-white shadow-sm' + : 'text-slate-600 dark:text-zinc-400 hover:text-slate-900 dark:hover:text-white' + }` + + return ( +
    +
    +
    +

    {t('importPage.title', 'Import')}

    +

    + {t('importPage.subtitle', 'Bring books you already have into Bindery.')} +

    +
    +
    + {VIEWS.map(v => ( + + ))} +
    +
    + +
    + {view === 'folder' ? ( + + ) : isAdmin ? ( + + ) : ( +

    + {t('adoption.adminOnly', 'Only an admin can decide what the files in the library are.')} +

    + )} +
    +
    + ) +} diff --git a/web/src/pages/import/adoptionGroups.ts b/web/src/pages/import/adoptionGroups.ts new file mode 100644 index 000000000..b67b6cc1b --- /dev/null +++ b/web/src/pages/import/adoptionGroups.ts @@ -0,0 +1,77 @@ +import type { AdoptionItem } from '../../api/client' + +// When several books on the page failed for the same reason, the author is +// not in the library and they share an author folder, that is one decision, +// not one per book. They are shown as a single group row with one Add author; +// the books stay reachable inside it. + +export interface AdoptionGroup { + kind: 'group' + key: string + folder: string + author: string + items: AdoptionItem[] + files: number +} + +export interface AdoptionEntry { + kind: 'item' + item: AdoptionItem + // Set when the row sits inside a group. + groupKey?: string +} + +export type AdoptionListEntry = AdoptionGroup | AdoptionEntry + +const MIN_GROUP = 2 + +function groupable(item: AdoptionItem): boolean { + return item.state === 'pending' && item.reason === 'author_not_in_library' && item.authorFolder !== '' && item.candidates.length === 0 +} + +// groupEntries keeps the page order: a group takes the place of its first book. +export function groupEntries(items: AdoptionItem[]): AdoptionListEntry[] { + const byFolder = new Map() + for (const item of items) { + if (!groupable(item)) continue + const list = byFolder.get(item.authorFolder) ?? [] + list.push(item) + byFolder.set(item.authorFolder, list) + } + const out: AdoptionListEntry[] = [] + const placed = new Set() + for (const item of items) { + const members = groupable(item) ? byFolder.get(item.authorFolder) : undefined + if (!members || members.length < MIN_GROUP) { + out.push({ kind: 'item', item }) + continue + } + if (placed.has(item.authorFolder)) continue + placed.add(item.authorFolder) + const authorCounts = new Map() + for (const m of members) if (m.parsedAuthor) authorCounts.set(m.parsedAuthor, (authorCounts.get(m.parsedAuthor) ?? 0) + 1) + const author = [...authorCounts.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? item.authorFolder + out.push({ + kind: 'group', + key: `group:${item.authorFolder}`, + folder: item.authorFolder, + author, + items: members, + files: members.reduce((n, m) => n + m.fileCount, 0), + }) + } + return out +} + +// visibleEntries flattens groups for rendering and keyboard order: a group +// row, then its books when it is open. +export function visibleEntries(entries: AdoptionListEntry[], open: ReadonlySet): AdoptionListEntry[] { + const out: AdoptionListEntry[] = [] + for (const e of entries) { + out.push(e) + if (e.kind === 'group' && open.has(e.key)) { + for (const item of e.items) out.push({ kind: 'item', item, groupKey: e.key }) + } + } + return out +} diff --git a/web/src/pages/import/adoptionHint.test.ts b/web/src/pages/import/adoptionHint.test.ts new file mode 100644 index 000000000..bc6d3e892 --- /dev/null +++ b/web/src/pages/import/adoptionHint.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from 'vitest' +import type { TFunction } from 'i18next' +import en from '../../i18n/locales/en.json' +import type { AdoptionItem } from '../../api/client' +import { adoptionHint, scorePercent } from './adoptionHint' +import { STRONG_MATCH_SCORE, authorsMatch, matchStrength } from './adoptionMatch' + +const lookup = (key: string): unknown => + key.split('.').reduce((n, p) => (n && typeof n === 'object' ? (n as Record)[p] : undefined), en) + +const t = ((key: string, options?: string | Record) => { + const opts = typeof options === 'object' ? options : {} + const value = lookup(key) + return (typeof value === 'string' ? value : key).replace(/\{\{(\w+)\}\}/g, (_, k) => String(opts[k] ?? '')) +}) as unknown as TFunction + +function item(overrides: Partial): AdoptionItem { + return { + id: 1, kind: 'file', format: 'ebook', fileCount: 1, sizeBytes: 1, relPath: 'A/B.epub', rootPath: '/books', + authorFolder: 'A', parsedTitle: 'B', parsedAuthor: 'Becky Chambers', reason: 'no_title_match', candidates: [], + topScore: 0, state: 'pending', bookCreated: false, authorCreated: false, members: [], firstSeenAt: '', + ...overrides, + } +} + +// The row's sentence is written from facts and ends in an action; the raw +// reason code only ever appears in the tooltip (#2033). +describe('adoptionHint', () => { + const codes = ['author_not_in_library', 'no_candidate_books', 'no_title_match', 'no_title_parsed'] as const + + it.each(codes)('never puts the %s code in the sentence, and ends in an action', code => { + const hint = adoptionHint(item({ reason: code }), t) + expect(hint.sentence).not.toContain(code) + expect(hint.sentence).toMatch(/(Add the author, then scan again|Choose the book[^.]*|search metadata)\.$/) + expect(hint.tooltip).toContain(code) + }) + + it('names the author the scan read when the author is missing', () => { + expect(adoptionHint(item({ reason: 'author_not_in_library' }), t).sentence) + .toBe('Becky Chambers is not in your library yet. Add the author, then scan again.') + }) + + const martian = { id: 9, title: 'The Martian', authorId: 1, authorName: 'Andy Weir', status: 'wanted', mediaType: 'ebook', monitored: true } + + it('calls a close title by the same author a strong match', () => { + const strong = item({ parsedAuthor: 'Andy Weir', candidates: [{ book: martian, score: 0.95 }] }) + expect(matchStrength(strong)).toBe('strong') + expect(adoptionHint(strong, t).sentence).toBe('Strong match: The Martian by Andy Weir. Confirm it or choose another book.') + }) + + it('calls anything weaker a possible match, never a one click confirm', () => { + // 0.82 is the "A Martyrs Tale" against "The Martian" case. + const weak = item({ parsedAuthor: 'Andy Weir', candidates: [{ book: martian, score: 0.82 }] }) + expect(matchStrength(weak)).toBe('possible') + expect(adoptionHint(weak, t).sentence).not.toContain('Confirm') + // A near identical title by another author is still only possible. + expect(matchStrength(item({ parsedAuthor: 'Ann Leckie', candidates: [{ book: martian, score: 0.99 }] }))).toBe('possible') + expect(STRONG_MATCH_SCORE).toBe(0.92) + }) + + it('matches authors by their words', () => { + expect(authorsMatch('Weir', 'Andy Weir')).toBe(true) + expect(authorsMatch('Álvaro Enrigue', 'Alvaro Enrigue')).toBe(true) + expect(authorsMatch('Andy Weir', 'Ann Leckie')).toBe(false) + expect(authorsMatch('', 'Andy Weir')).toBe(false) + }) + + it('clamps scores to a percentage', () => { + expect([scorePercent(-1), scorePercent(0.6), scorePercent(1.4)]).toEqual([0, 60, 100]) + }) +}) diff --git a/web/src/pages/import/adoptionHint.ts b/web/src/pages/import/adoptionHint.ts new file mode 100644 index 000000000..7a665bbd7 --- /dev/null +++ b/web/src/pages/import/adoptionHint.ts @@ -0,0 +1,94 @@ +import type { TFunction } from 'i18next' +import type { AdoptionItem } from '../../api/client' +import { matchStrength } from './adoptionMatch' + +// The one sentence a row says about itself. It is written from facts the scan +// recorded (the parsed author, the closest title) and ends in the thing to do +// next. The scanner's reason code is only ever a tooltip: the codes are exact +// but read as blame, and "no title match" was what sent #2033's reporter off +// renaming files that were fine. + +export interface AdoptionHintText { + sentence: string + tooltip: string +} + +// A score is a 0 to 1 Jaro-Winkler similarity; people read a percentage. +export function scorePercent(score: number): number { + return Math.round(Math.max(0, Math.min(1, score)) * 100) +} + +export function adoptionHint(item: AdoptionItem, t: TFunction): AdoptionHintText { + const tooltip = item.reason + ? t('adoption.hint.reasonTooltip', { code: item.reason, defaultValue: 'Scan reason: {{code}}' }) + : '' + const author = item.parsedAuthor + const top = item.candidates[0] + + if (top) { + const strong = matchStrength(item) === 'strong' + return { + sentence: strong + ? t('adoption.hint.strong', { + title: top.book.title, author: top.book.authorName, + defaultValue: 'Strong match: {{title}} by {{author}}. Confirm it or choose another book.', + }) + : t('adoption.hint.possible', { + title: top.book.title, author: top.book.authorName, percent: scorePercent(top.score), + defaultValue: 'Possible match: {{title}} by {{author}}, {{percent}}% title similarity. Check it, then adopt it or choose another book.', + }), + tooltip, + } + } + switch (item.reason) { + case 'author_not_in_library': + return { + sentence: author + ? t('adoption.hint.authorMissing', { author, defaultValue: '{{author}} is not in your library yet. Add the author, then scan again.' }) + : t('adoption.hint.authorUnreadable', 'No author could be read from these files. Choose the book they are.'), + tooltip, + } + case 'no_candidate_books': + return { + sentence: t('adoption.hint.noCandidates', { + author: author || item.authorFolder, + defaultValue: '{{author}} has no book in your library waiting for these files. Choose the book or search metadata.', + }), + tooltip, + } + case 'no_title_parsed': + return { + sentence: t('adoption.hint.noTitle', 'No title could be read from the file names or tags. Choose the book they are.'), + tooltip, + } + default: + return { + sentence: author + ? t('adoption.hint.noTitleMatch', { author, defaultValue: 'No book by {{author}} has a title close to this one. Choose the book it is.' }) + : t('adoption.hint.noTitleMatchNoAuthor', 'No book in your library has a title close to this one. Choose the book it is.'), + tooltip, + } + } +} + +// The name a row goes by: the title the scan read, else the last part of its +// path. +export function unitDisplayName(item: AdoptionItem): string { + if (item.parsedTitle) return item.parsedTitle + const parts = item.relPath.split('/') + return parts[parts.length - 1] || item.relPath +} + +// "2 hours ago", in the reader's language. +export function relativeTime(iso: string | undefined, locale?: string): string { + if (!iso) return '' + const then = Date.parse(iso) + if (Number.isNaN(then)) return '' + const seconds = Math.round((then - Date.now()) / 1000) + const rtf = new Intl.RelativeTimeFormat(locale, { numeric: 'auto' }) + const steps: [Intl.RelativeTimeFormatUnit, number][] = [['day', 86400], ['hour', 3600], ['minute', 60]] + for (const [unit, size] of steps) { + if (Math.abs(seconds) >= size) return rtf.format(Math.round(seconds / size), unit) + } + return rtf.format(0, 'minute') +} diff --git a/web/src/pages/import/adoptionMatch.ts b/web/src/pages/import/adoptionMatch.ts new file mode 100644 index 000000000..2dab50174 --- /dev/null +++ b/web/src/pages/import/adoptionMatch.ts @@ -0,0 +1,58 @@ +import type { TFunction } from 'i18next' +import type { AdoptionCandidate, AdoptionItem } from '../../api/client' + +// How much a suggestion deserves. Only a strong one gets a one click Confirm: +// a close title alone is not enough ("A Martyrs Tale" scores 0.83 against "The +// Martian"), so the author the scan read must also be the book's author. +// Everything below is a possible match that a person looks at first. + +// STRONG_MATCH_SCORE is the title similarity (0 to 1) a suggestion needs, with +// a matching author, to be offered as a one click Confirm. +export const STRONG_MATCH_SCORE = 0.92 + +export type MatchStrength = 'strong' | 'possible' + +function nameTokens(name: string): string[] { + return name + .normalize('NFKD') + .replace(/\p{M}/gu, '') + .toLowerCase() + .split(/[^\p{L}\p{N}]+/u) + .filter(w => w.length > 1) +} + +// authorsMatch reports whether every significant word of the author the scan +// read appears in the book's author, so "Weir" and "Andy Weir" match and +// "Andy Weir" and "Ann Leckie" do not. An unreadable author never matches. +export function authorsMatch(parsedAuthor: string, bookAuthor: string): boolean { + const parsed = nameTokens(parsedAuthor) + if (parsed.length === 0) return false + const book = new Set(nameTokens(bookAuthor)) + return parsed.every(w => book.has(w)) +} + +export function matchStrength(item: AdoptionItem, candidate: AdoptionCandidate | undefined = item.candidates[0]): MatchStrength | null { + if (!candidate) return null + const author = item.parsedAuthor || item.authorFolder + return candidate.score >= STRONG_MATCH_SCORE && authorsMatch(author, candidate.book.authorName) ? 'strong' : 'possible' +} + +// shortHint is the row's one line: the fact, without the advice, which lives +// in the tooltip and the editor. +export function shortHint(item: AdoptionItem, t: TFunction): string { + const author = item.parsedAuthor || item.authorFolder + switch (item.reason) { + case 'author_not_in_library': + return author + ? t('adoption.short.authorMissing', { author, defaultValue: '{{author}} is not in your library' }) + : t('adoption.short.authorUnreadable', 'No author could be read') + case 'no_candidate_books': + return t('adoption.short.noCandidates', { author, defaultValue: 'No book by {{author}} is waiting for a file' }) + case 'no_title_parsed': + return t('adoption.short.noTitle', 'No title could be read') + default: + return author + ? t('adoption.short.noTitleMatch', { author, defaultValue: 'No close title by {{author}}' }) + : t('adoption.short.noTitleMatchNoAuthor', 'No close title in your library') + } +} diff --git a/web/src/pages/import/adoptionReducer.ts b/web/src/pages/import/adoptionReducer.ts new file mode 100644 index 000000000..42f5185c6 --- /dev/null +++ b/web/src/pages/import/adoptionReducer.ts @@ -0,0 +1,119 @@ +import type { + AdoptionBookRef, AdoptionFacets, AdoptionItem, AdoptionListResponse, AdoptionScanStatus, AdoptionSummary, +} from '../../api/client' + +// Optimistic state for the adoption list, as named actions. A decision shows +// at once as a quiet line on its row; the server's answer then confirms it or +// reverts it with the error inline. The line stays until the next fetch, so a +// person can read what happened and undo it without the row jumping away. + +export type Outcome = + // Adopt sent; preview is the book the person picked, shown before the answer. + | { kind: 'adopting'; preview: AdoptionBookRef | null } + | { kind: 'adopted'; item: AdoptionItem } + | { kind: 'ignoring' } + | { kind: 'ignored' } + | { kind: 'undoing' } + // Undone or unignored while viewing the adopted or ignored list: the row + // has gone back to needing a decision. + | { kind: 'restored' } + +export interface AdoptionListState { + items: AdoptionItem[] + total: number + facets: AdoptionFacets | null + summary: AdoptionSummary | null + scan: AdoptionScanStatus | null + loading: boolean + loaded: boolean + loadError: string + outcomes: Record + errors: Record + // Quiet notes on rows, such as Undo keeping a book that is in use. Cleared + // with the next fetch, like outcomes. + notes: Record + expandedId: number | null +} + +export const initialAdoptionState: AdoptionListState = { + items: [], total: 0, facets: null, summary: null, scan: null, + loading: true, loaded: false, loadError: '', outcomes: {}, errors: {}, notes: {}, expandedId: null, +} + +export type AdoptionAction = + | { type: 'loadStarted' } + | { type: 'loaded'; response: AdoptionListResponse } + | { type: 'loadFailed'; error: string } + | { type: 'expanded'; id: number } + | { type: 'collapsed' } + | { type: 'adoptRequested'; id: number; preview: AdoptionBookRef | null } + | { type: 'adoptSucceeded'; id: number; item: AdoptionItem } + | { type: 'ignoreRequested'; id: number } + | { type: 'ignoreSucceeded'; id: number } + | { type: 'undoRequested'; id: number } + | { type: 'undoSucceeded'; id: number; item: AdoptionItem; restored: boolean; keptBook: boolean } + | { type: 'requestFailed'; id: number; error: string; revertTo: Outcome | null } + | { type: 'scanStatus'; summary: AdoptionSummary; scan: AdoptionScanStatus } + +function withOutcome(state: AdoptionListState, id: number, outcome: Outcome | null): AdoptionListState { + const outcomes = { ...state.outcomes } + if (outcome) outcomes[id] = outcome + else delete outcomes[id] + const errors = { ...state.errors } + delete errors[id] + return { ...state, outcomes, errors } +} + +export function adoptionReducer(state: AdoptionListState, action: AdoptionAction): AdoptionListState { + switch (action.type) { + case 'loadStarted': + return { ...state, loading: true, loadError: '' } + case 'loaded': { + const { response } = action + return { + ...state, + loading: false, + loaded: true, + items: response.items, + total: response.total, + // Facets come only with a filter change; a page turn keeps the last. + facets: response.facets ?? state.facets, + summary: response.summary, + scan: response.scan, + outcomes: {}, + errors: {}, + notes: {}, + expandedId: response.items.some(i => i.id === state.expandedId) ? state.expandedId : null, + } + } + case 'loadFailed': + return { ...state, loading: false, loaded: true, loadError: action.error } + case 'expanded': + return { ...state, expandedId: action.id } + case 'collapsed': + return { ...state, expandedId: null } + case 'adoptRequested': + return { ...withOutcome(state, action.id, { kind: 'adopting', preview: action.preview }), expandedId: null } + case 'adoptSucceeded': + return withOutcome(state, action.id, { kind: 'adopted', item: action.item }) + case 'ignoreRequested': + return { ...withOutcome(state, action.id, { kind: 'ignoring' }), expandedId: null } + case 'ignoreSucceeded': + return withOutcome(state, action.id, { kind: 'ignored' }) + case 'undoRequested': + return withOutcome(state, action.id, { kind: 'undoing' }) + case 'undoSucceeded': { + const next = withOutcome(state, action.id, action.restored ? { kind: 'restored' } : null) + const notes = { ...state.notes } + if (action.keptBook) notes[action.id] = 'keptBook' + else delete notes[action.id] + return { ...next, notes, items: state.items.map(i => (i.id === action.id ? action.item : i)) } + } + case 'requestFailed': { + const next = withOutcome(state, action.id, action.revertTo) + return { ...next, errors: { ...next.errors, [action.id]: action.error } } + } + case 'scanStatus': + return { ...state, summary: action.summary, scan: action.scan } + } +} diff --git a/web/src/pages/import/adoptionStyles.ts b/web/src/pages/import/adoptionStyles.ts new file mode 100644 index 000000000..9806c6e8d --- /dev/null +++ b/web/src/pages/import/adoptionStyles.ts @@ -0,0 +1,16 @@ +// Shared row geometry for the adoption table, so a book row, a group row and +// an editor row line up and every row has the same height at desktop width. + +export const cellCls = 'block md:table-cell px-3 py-1 md:py-2.5 align-middle min-w-0 md:max-w-0' + +// The action cell has a fixed width: one primary button and the More menu. +export const actionCellCls = 'block md:table-cell px-3 pb-3 pt-1 md:py-2.5 align-middle md:w-52 md:whitespace-nowrap' + +export function rowCls(expanded: boolean, inGroup: boolean): string { + const surface = expanded + ? 'bg-emerald-500/5' + : inGroup + ? 'bg-white dark:bg-zinc-950/40 hover:bg-slate-100 dark:hover:bg-zinc-900' + : 'bg-slate-100/50 dark:bg-zinc-900/50 hover:bg-slate-200/50 dark:hover:bg-zinc-800/50' + return `block md:table-row mb-3 md:mb-0 rounded-lg md:rounded-none border md:border-0 border-slate-200 dark:border-zinc-800 outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-emerald-500 ${inGroup ? 'ml-4 md:ml-0' : ''} ${surface}` +} diff --git a/web/src/pages/import/folderImport.ts b/web/src/pages/import/folderImport.ts new file mode 100644 index 000000000..52d3f1392 --- /dev/null +++ b/web/src/pages/import/folderImport.ts @@ -0,0 +1,23 @@ +import type { Book, ScanItem } from '../../api/client' + +// Shared by FolderImportView and FolderImportRow. + +// RowState is the per-unit resolution the user is building, keyed by unit path. +export interface RowState { + // chosen is the catalogue book the unit will import against (a confident + // match, a picked candidate, or a searched existing book). null until resolved. + chosen: Book | null + // format override: '' = auto-detect, or 'ebook' / 'audiobook'. Defaults to the + // scan's detected format. + format: string + // selected marks the unit for the next bulk import. Only meaningful when chosen. + selected: boolean +} + +export function groupHeading(match: ScanItem['match']): string { + switch (match) { + case 'confident': return 'Matched' + case 'ambiguous': return 'Needs a choice' + default: return 'Unmatched' + } +} diff --git a/web/src/pages/import/useAdoptionList.ts b/web/src/pages/import/useAdoptionList.ts new file mode 100644 index 000000000..d96393dae --- /dev/null +++ b/web/src/pages/import/useAdoptionList.ts @@ -0,0 +1,173 @@ +import { useCallback, useEffect, useReducer, useRef, useState } from 'react' +import { api } from '../../api/client' +import type { AdoptionItem, AdoptionSort, AdoptionState, AdoptTarget, AdoptionBookRef } from '../../api/client' +import { useServerPagination } from '../../components/usePagination' +import { adoptionReducer, initialAdoptionState } from './adoptionReducer' +import { notifyUnmatchedChanged } from '../../components/useUnmatchedCount' + +// useAdoptionList owns the adoption view's data: filters, paging, the fetch, +// optimistic decisions (adoptionReducer) and the scan poll. Components only +// render what it returns and call its verbs. + +export interface AdoptionFilters { + state: AdoptionState + reason: string + authorFolder: string + format: string + search: string + sort: AdoptionSort + dir: '' | 'asc' | 'desc' +} + +export const defaultAdoptionFilters: AdoptionFilters = { + state: 'pending', reason: '', authorFolder: '', format: '', search: '', sort: 'score', dir: '', +} + +// How often the page asks for the scan status while a scan runs (P5). Nothing +// polls otherwise. +export const SCAN_POLL_MS = 3000 + +function message(e: unknown, fallback: string): string { + return e instanceof Error && e.message ? e.message : fallback +} + +export function useAdoptionList() { + const [state, dispatch] = useReducer(adoptionReducer, initialAdoptionState) + const [filters, setFilters] = useState(defaultAdoptionFilters) + const [debouncedSearch, setDebouncedSearch] = useState('') + const { page, pageSize, paginationProps, reset } = useServerPagination(state.total, 50, 'adoption') + const request = useRef(0) + // Facets are fetched once per state and search, not on every page turn or + // filter pick (P4); the key records what the held facets were computed for. + const facetKey = useRef('') + const [reloadTick, setReloadTick] = useState(0) + + useEffect(() => { + const id = setTimeout(() => setDebouncedSearch(filters.search.trim()), 300) + return () => clearTimeout(id) + }, [filters.search]) + + useEffect(() => { reset() }, [filters.state, filters.reason, filters.authorFolder, filters.format, debouncedSearch, filters.sort, filters.dir, reset]) + + useEffect(() => { + const n = ++request.current + const key = `${filters.state}|${debouncedSearch}|${reloadTick}` + const wantFacets = key !== facetKey.current + dispatch({ type: 'loadStarted' }) + api.listUnmatched({ + state: filters.state, + reason: filters.reason || undefined, + authorFolder: filters.authorFolder || undefined, + format: filters.format || undefined, + search: debouncedSearch || undefined, + sort: filters.sort, + dir: filters.dir || undefined, + limit: pageSize, + offset: (page - 1) * pageSize, + facets: wantFacets, + }).then(response => { + if (n !== request.current) return + if (wantFacets) facetKey.current = key + dispatch({ type: 'loaded', response }) + }).catch(e => { + if (n === request.current) dispatch({ type: 'loadFailed', error: message(e, 'Could not load the list') }) + }) + }, [filters.state, filters.reason, filters.authorFolder, filters.format, filters.sort, filters.dir, debouncedSearch, page, pageSize, reloadTick]) + + const reload = useCallback(() => setReloadTick(n => n + 1), []) + + const refreshSummary = useCallback(async () => { + try { + const { scan, ...summary } = await api.unmatchedSummary() + dispatch({ type: 'scanStatus', summary, scan }) + notifyUnmatchedChanged(summary.pending) + return scan + } catch { + return null + } + }, []) + + // Poll only while a scan runs; reload the list once it finishes. + const running = Boolean(state.scan?.running) + useEffect(() => { + if (!running) return + const id = setInterval(async () => { + const scan = await refreshSummary() + if (scan && !scan.running) reload() + }, SCAN_POLL_MS) + return () => clearInterval(id) + }, [running, refreshSummary, reload]) + + const setFilter = useCallback((patch: Partial) => setFilters(f => ({ ...f, ...patch })), []) + + const adopt = useCallback(async (item: AdoptionItem, target: AdoptTarget, preview: AdoptionBookRef | null) => { + dispatch({ type: 'adoptRequested', id: item.id, preview }) + try { + const updated = await api.adoptUnit(item.id, target) + dispatch({ type: 'adoptSucceeded', id: item.id, item: updated }) + void refreshSummary() + } catch (e) { + dispatch({ type: 'requestFailed', id: item.id, error: message(e, 'Could not adopt this book'), revertTo: null }) + } + }, [refreshSummary]) + + const ignore = useCallback(async (item: AdoptionItem) => { + dispatch({ type: 'ignoreRequested', id: item.id }) + try { + await api.ignoreUnit(item.id) + dispatch({ type: 'ignoreSucceeded', id: item.id }) + void refreshSummary() + } catch (e) { + dispatch({ type: 'requestFailed', id: item.id, error: message(e, 'Could not ignore this book'), revertTo: null }) + } + }, [refreshSummary]) + + // undo reverses whatever the row's last decision was: an adoption (undo) or + // an ignore (unignore). On the adopted and ignored lists the row then reads + // as restored; on the pending list it simply needs a decision again. + const undo = useCallback(async (item: AdoptionItem) => { + const prior = state.outcomes[item.id] ?? null + const wasIgnored = prior?.kind === 'ignored' || (!prior && item.state === 'ignored') + dispatch({ type: 'undoRequested', id: item.id }) + try { + const updated = wasIgnored ? await api.unignoreUnit(item.id) : await api.undoAdoption(item.id) + dispatch({ type: 'undoSucceeded', id: item.id, item: updated, restored: filters.state !== 'pending', keptBook: Boolean(updated.message) }) + void refreshSummary() + } catch (e) { + dispatch({ type: 'requestFailed', id: item.id, error: message(e, 'Could not undo'), revertTo: prior }) + } + }, [state.outcomes, filters.state, refreshSummary]) + + const startScan = useCallback(async () => { + try { + await api.triggerLibraryScan() + } catch { + // 409: a scan is already running, which is what we wanted anyway. + } + await refreshSummary() + }, [refreshSummary]) + + const ignoreFolder = useCallback(async (folder: string) => { + await api.ignoreFolder(folder) + await refreshSummary() + setFilters(f => (f.authorFolder === folder ? { ...f, authorFolder: '' } : f)) + reload() + }, [refreshSummary, reload]) + + return { + state, + filters, + setFilter, + paginationProps, + expand: (id: number) => dispatch({ type: 'expanded', id }), + collapse: () => dispatch({ type: 'collapsed' }), + adopt, + ignore, + undo, + startScan, + ignoreFolder, + reload, + } +} + +export type AdoptionList = ReturnType diff --git a/web/src/pages/settings/FolderScanSection.test.tsx b/web/src/pages/settings/FolderScanSection.test.tsx deleted file mode 100644 index 363df318d..000000000 --- a/web/src/pages/settings/FolderScanSection.test.tsx +++ /dev/null @@ -1,106 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen, fireEvent, waitFor } from '@testing-library/react' - -// i18n: return the key (with crude interpolation) so assertions are stable. -vi.mock('react-i18next', () => ({ - useTranslation: () => ({ - t: (key: string, options?: unknown) => { - // t(key) and t(key, 'default string') both render the bare key for - // stable assertions; t(key, {opts}) appends the interpolation values. - if (!options || typeof options !== 'object') return key - let out = key - for (const [k, v] of Object.entries(options as Record)) { - if (k === 'defaultValue') continue - out += ` ${k}=${String(v)}` - } - return out - }, - }), -})) - -vi.mock('../../api/client', () => ({ - api: { - scanFolder: vi.fn(), - batchImport: vi.fn(), - }, -})) - -import { api, FolderScanResponse } from '../../api/client' -import { FolderScanSection } from './ImportTab' - -const mockScan = api.scanFolder as ReturnType -const mockBatch = api.batchImport as ReturnType - -function scanResult(): FolderScanResponse { - return { - truncated: false, - items: [ - { - path: '/dl/Confident Book', name: 'Confident Book', match: 'confident', - parsedTitle: 'Confident Book', parsedAuthor: 'A. Writer', detectedFormat: 'ebook', - book: { id: 11, title: 'Confident Book', author: { authorName: 'A. Writer' } } as never, - }, - { - path: '/dl/Maybe Book', name: 'Maybe Book', match: 'ambiguous', - parsedTitle: 'Maybe Book', parsedAuthor: '', detectedFormat: 'ebook', - candidates: [ - { id: 21, title: 'Maybe Book (1)', author: { authorName: 'X' } } as never, - { id: 22, title: 'Maybe Book (2)', author: { authorName: 'Y' } } as never, - ], - }, - { - path: '/dl/Orphan', name: 'Orphan', match: 'none', - parsedTitle: 'Orphan', parsedAuthor: '', detectedFormat: 'audiobook', - }, - ], - } -} - -describe('FolderScanSection', () => { - beforeEach(() => vi.clearAllMocks()) - - async function scan() { - mockScan.mockResolvedValue(scanResult()) - render() - fireEvent.change(screen.getByPlaceholderText('settings.import.bulkPathPlaceholder'), { - target: { value: '/dl' }, - }) - fireEvent.click(screen.getByText('settings.import.bulkScan')) - await waitFor(() => expect(screen.getByText('Confident Book')).toBeInTheDocument()) - } - - it('pre-selects only confident matches and imports them', async () => { - await scan() - expect(mockScan).toHaveBeenCalledWith('/dl') - - const checkboxes = screen.getAllByRole('checkbox') as HTMLInputElement[] - expect(checkboxes[0].checked).toBe(true) // confident - expect(checkboxes[1].checked).toBe(false) // ambiguous, no pick yet - expect(checkboxes[2].disabled).toBe(true) // none, not selectable - - // Import button reflects 1 selected (the confident one). - expect(screen.getByText(/settings\.import\.bulkImport count=1/)).toBeInTheDocument() - - mockBatch.mockResolvedValue({ accepted: 1, failed: 0, results: [{ path: '/dl/Confident Book', accepted: true, downloadId: 5 }] }) - fireEvent.click(screen.getByText(/settings\.import\.bulkImport count=1/)) - - await waitFor(() => expect(mockBatch).toHaveBeenCalledTimes(1)) - expect(mockBatch).toHaveBeenCalledWith([{ path: '/dl/Confident Book', bookId: 11, format: 'ebook' }]) - await waitFor(() => expect(screen.getByText(/bulkSummary accepted=1 failed=0/)).toBeInTheDocument()) - }) - - it('lets an ambiguous match be resolved via the picker and included', async () => { - await scan() - // Pick candidate 22 for the ambiguous row. - fireEvent.change(screen.getByRole('combobox'), { target: { value: '22' } }) - expect(screen.getByText(/settings\.import\.bulkImport count=2/)).toBeInTheDocument() - - mockBatch.mockResolvedValue({ accepted: 2, failed: 0, results: [] }) - fireEvent.click(screen.getByText(/settings\.import\.bulkImport count=2/)) - await waitFor(() => expect(mockBatch).toHaveBeenCalledTimes(1)) - expect(mockBatch).toHaveBeenCalledWith([ - { path: '/dl/Confident Book', bookId: 11, format: 'ebook' }, - { path: '/dl/Maybe Book', bookId: 22, format: 'ebook' }, - ]) - }) -}) diff --git a/web/src/pages/settings/GeneralTab.tsx b/web/src/pages/settings/GeneralTab.tsx index edabf99bc..e67d462be 100644 --- a/web/src/pages/settings/GeneralTab.tsx +++ b/web/src/pages/settings/GeneralTab.tsx @@ -1,7 +1,7 @@ import { FormEvent, useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { useConfirmDialog } from '../../components/useConfirmDialog' -import { api, AuthConfig, AuthStatus, StorageDirStatus, StorageHealth } from '../../api/client' +import { api, AuthConfig, AuthStatus, BINDERY_BASE, StorageDirStatus, StorageHealth } from '../../api/client' import AuthSettings from '../../settings/AuthSettings' import ThemeToggle from '../../components/ThemeToggle' import LanguageSwitcher from '../../components/LanguageSwitcher' @@ -42,9 +42,10 @@ export default function GeneralTab({ onNavigate }: GeneralTabProps = {}) { unmatched: number already_tracked?: number tag_read_failed?: number - // reason is one of the scanner's unmatchedReason* codes (#1958); absent on - // scan results persisted before that field existed. - unmatched_files?: Array<{ path: string; parsed_title: string; parsed_author: string; reason?: string }> + // Books (not files) the scan could not match, and how many are ignored. + // Absent on results persisted before library adoption. + unmatched_units?: number + ignored_units?: number library_dir?: string audiobook_dir?: string scanned_paths?: string[] @@ -560,61 +561,17 @@ export default function GeneralTab({ onNavigate }: GeneralTabProps = {}) { })}

    )} - {lastScan.files_found > 0 && lastScan.unmatched > 0 && lastScan.reconciled === 0 && ( -

    - {(() => { - // The catalogue advice is only right when the author matched - // but has no book to attach a file to. When every unmatched - // file's parsed author matches NO author in the library, - // refreshing an author cannot help — name the parsed author - // instead, which is the evidence to act on (#1958). - // unmatched_files is a sample capped at 1000 by the scanner; - // every() over it says nothing about the rest, so a run with - // 5000 unmatched files whose first 1000 happen to share a - // reason must not claim the diagnosis holds library-wide. - const files = lastScan.unmatched_files ?? [] - const truncated = files.length < lastScan.unmatched - if (truncated || files.length === 0 || !files.every(f => f.reason === 'author_not_in_library')) { - return t('settings.general.scanAllUnmatchedHint') - } - const authors = Array.from(new Set(files.map(f => f.parsed_author).filter(Boolean))) - return authors.length === 1 - ? t('settings.general.scanUnmatchedAuthorUnknown', { author: authors[0] }) - : t('settings.general.scanUnmatchedAuthorsUnknown') - })()} + {/* The unmatched files used to be listed here, capped at 1000 + per file rows with no way to act on one. They are books to + decide about on the Import page now; this only counts them. */} + {(lastScan.unmatched_units ?? 0) > 0 && ( +

    + {t('settings.general.unmatchedUnits', { count: lastScan.unmatched_units, defaultValue: '{{count}} books need a decision.' })} + + {t('settings.general.reviewUnmatched', { count: lastScan.unmatched_units, defaultValue: 'Review {{count}} unmatched books' })} +

    )} - {lastScan.unmatched_files && lastScan.unmatched_files.length > 0 && ( -
    - - Unmatched files ({lastScan.unmatched_files.length}{lastScan.unmatched_files.length >= 1000 && lastScan.unmatched > 1000 ? ' of ' + lastScan.unmatched : ''}) - -
    - - - - - - - - - - - {lastScan.unmatched_files.map((file, idx) => ( - - - - - {/* Per-file diagnosis from the scanner (#1958). Blank - for results persisted before the field existed. */} - - - ))} - -
    {t('settings.general.scanUnmatchedPath')}{t('settings.general.scanUnmatchedParsedTitle')}{t('settings.general.scanUnmatchedParsedAuthor')}{t('settings.general.scanUnmatchedWhy')}
    {file.path}{file.parsed_title || '—'}{file.parsed_author || '—'}{file.reason ? t(`settings.general.scanReason.${file.reason}`) : '—'}
    -
    -
    - )}
    )}
    diff --git a/web/src/pages/settings/ImportTab.tsx b/web/src/pages/settings/ImportTab.tsx index fce364968..eee2c0555 100644 --- a/web/src/pages/settings/ImportTab.tsx +++ b/web/src/pages/settings/ImportTab.tsx @@ -1,6 +1,6 @@ import { useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' -import { api, BatchImportItem, BatchImportResponse, Book, HardcoverList, ImportList, ImportListSyncProgress, ManagedUser, ManualImportLookup, ScanItem } from '../../api/client' +import { api, BINDERY_BASE, Book, HardcoverList, ImportList, ImportListSyncProgress, ManagedUser, ManualImportLookup } from '../../api/client' import { inputCls } from './formStyles' import GoodreadsImportSection from './GoodreadsImportSection' @@ -118,7 +118,7 @@ export default function ImportTab({ onNavigate }: ImportTabProps = {}) { )} - + @@ -263,198 +263,22 @@ function ManualImportSection() { ) } -interface ScanRowState { - include: boolean - bookId: number | null - format: string -} - -// FolderScanSection scans a folder for book units and bulk-imports the selected -// matches in one shot, so a migration backlog doesn't have to be imported one -// path at a time. Matches against the existing catalogue (add the authors/books -// first); unmatched units are listed but cannot be selected. Exported for tests. -export function FolderScanSection() { +// FolderImportLink stands where the bulk folder scan used to be. That scan +// now lives on the Import page, next to the view for files already in the +// library, so there is one place to bring books in rather than two. +function FolderImportLink() { const { t } = useTranslation() - const [path, setPath] = useState('') - const [scanning, setScanning] = useState(false) - const [items, setItems] = useState(null) - const [truncated, setTruncated] = useState(false) - const [rows, setRows] = useState([]) - const [importing, setImporting] = useState(false) - const [summary, setSummary] = useState(null) - const [err, setErr] = useState(null) - - const handleScan = async () => { - if (!path.trim()) return - setScanning(true) - setErr(null) - setItems(null) - setSummary(null) - try { - const r = await api.scanFolder(path.trim()) - setItems(r.items) - setTruncated(r.truncated) - setRows(r.items.map(it => ({ - include: it.match === 'confident', - bookId: it.match === 'confident' && it.book ? it.book.id : null, - format: it.detectedFormat || '', - }))) - } catch (e) { - setErr(e instanceof Error ? e.message : 'Scan failed') - } finally { - setScanning(false) - } - } - - const patchRow = (i: number, patch: Partial) => - setRows(prev => prev.map((r, idx) => (idx === i ? { ...r, ...patch } : r))) - - const selectedCount = rows.filter(r => r.include && r.bookId).length - - const handleImport = async () => { - if (!items) return - const batch: BatchImportItem[] = [] - rows.forEach((r, i) => { - if (r.include && r.bookId) batch.push({ path: items[i].path, bookId: r.bookId, format: r.format || undefined }) - }) - if (batch.length === 0) return - setImporting(true) - setErr(null) - try { - const res = await api.batchImport(batch) - setSummary(res) - } catch (e) { - setErr(e instanceof Error ? e.message : 'Import failed') - } finally { - setImporting(false) - } - } - - const matchBadge = (m: ScanItem['match']) => { - const map: Record = { - confident: 'bg-emerald-100 dark:bg-emerald-950 text-emerald-700 dark:text-emerald-400', - ambiguous: 'bg-amber-100 dark:bg-amber-950 text-amber-700 dark:text-amber-400', - none: 'bg-slate-200 dark:bg-zinc-800 text-slate-500 dark:text-zinc-500', - } - return {t(`settings.import.bulkMatch_${m}`, m)} - } - return (
    -

    {t('settings.import.bulkHeading', 'Bulk folder import')}

    -

    - {t('settings.import.bulkDescription', 'Scan a folder (e.g. your download directory) and import every book it can match to your library in one go. Matching is against books already in your library, so add the authors first. Unmatched items are listed but skipped.')} -

    - -
    - { setPath(e.target.value); setItems(null); setSummary(null) }} - onKeyDown={e => { if (e.key === 'Enter') handleScan() }} - /> - +

    {t('settings.import.folderMovedHeading', 'Import from a folder')}

    +
    +

    + {t('settings.import.folderMovedBody', 'Scanning a folder and importing what it matches is on the Import page now, beside the books a library scan could not match.')} +

    + + {t('settings.import.folderMovedLink', 'Open Import')} +
    - - {items && items.length === 0 && ( -

    {t('settings.import.bulkEmpty', 'No book files or folders found here.')}

    - )} - - {items && items.length > 0 && !summary && ( -
    - {truncated && ( -

    {t('settings.import.bulkTruncated', 'Showing the first 1000 items; narrow the folder to see the rest.')}

    - )} -
    - {items.map((it, i) => { - const row = rows[i] - const disabled = it.match === 'none' - return ( -
    - patchRow(i, { include: e.target.checked })} - aria-label={t('settings.import.bulkSelect', { name: it.name, defaultValue: `Import ${it.name}` })} - /> -
    -
    - {it.name} - {matchBadge(it.match)} - {it.detectedFormat} -
    - {/* Full source path so the user can tell which file a match - (or the ambiguous picker below) refers to — the basename - alone is ambiguous across folders (#1435). */} -
    {it.path}
    -
    - {t('settings.import.bulkParsed', { title: it.parsedTitle || '?', author: it.parsedAuthor || '?', defaultValue: `parsed: ${it.parsedTitle || '?'} / ${it.parsedAuthor || '?'}` })} -
    - {it.match === 'confident' && it.book && ( -
    → {it.book.title}{it.book.author ? ` (${it.book.author.authorName})` : ''}
    - )} - {it.match === 'ambiguous' && ( - - )} - {it.match === 'none' && ( -
    {t('settings.import.bulkNoMatch', 'No catalogue match; add the book first.')}
    - )} -
    -
    - ) - })} -
    - -
    - )} - - {summary && ( -
    -
    {t('settings.import.bulkSummary', { accepted: summary.accepted, failed: summary.failed, defaultValue: `${summary.accepted} queued, ${summary.failed} failed` })}
    - {summary.failed > 0 && ( -
    - {t('settings.import.bulkShowFailures', { count: summary.failed, defaultValue: `Show ${summary.failed} failures` })} -
      - {summary.results.filter(r => !r.accepted).map(r => ( -
    • {r.path}: {r.error}
      • - ))} -
    -
    - )} -

    {t('settings.import.bulkSummaryNote', 'Imports run in the background; watch the Queue for progress.')}

    -
    - )} - - {err &&

    {err}

    }
    ) }