From 76344af38313187c7ae7764aa9a0655045b60c3d Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 01:12:45 -0300 Subject: [PATCH 01/18] refactor(api): split AddBook and author Create into HTTP shells and reusable cores Library adoption and request approval both need to add a book or an author without an HTTP request. Move the bodies of AddBook and Create into addBookCore (add_book_core.go) and createAuthorCore (create_author_core.go). The handlers now decode the body, call the core, and map its errors back to the exact statuses and bodies they returned before. Ownership still comes from the context. This is a pure move: no behaviour change, and the existing tests pass unmodified. addBookResult reports whether the book and the author were created by the call, which the later callers need for compensation. authors.go shrinks by about 480 lines (plan item C1). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- internal/api/add_book_core.go | 525 ++++++++++++++++++++++++++++ internal/api/authors.go | 540 ++--------------------------- internal/api/create_author_core.go | 229 ++++++++++++ 3 files changed, 782 insertions(+), 512 deletions(-) create mode 100644 internal/api/add_book_core.go create mode 100644 internal/api/create_author_core.go diff --git a/internal/api/add_book_core.go b/internal/api/add_book_core.go new file mode 100644 index 000000000..a899c8ca7 --- /dev/null +++ b/internal/api/add_book_core.go @@ -0,0 +1,525 @@ +package api + +import ( + "context" + "errors" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/vavallee/bindery/internal/auth" + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/indexer" + "github.com/vavallee/bindery/internal/models" +) + +// addBookParams is everything an Add Book needs, independent of HTTP. The +// AddBook handler fills it from the request body; other callers (library +// adoption, request approval) fill it themselves. Ownership is not a field: +// the core reads the acting user from ctx, exactly as the handler always has. +type addBookParams struct { + ForeignBookID string + ForeignAuthorID string + AuthorName string + SearchOnAdd bool + // MediaType optionally forces ebook, audiobook or both for the added + // book (#1397). Empty keeps the provider's media type, falling back to + // the default.media_type setting. + MediaType string +} + +// addBookResult is what an Add Book produced. +type addBookResult struct { + // Book is the library row the add resolved to, after the monitored and + // media type update. + Book *models.Book + // BookCreated is true when this call inserted the row: the direct insert + // created it, or the single work fallback sync ran and the row appeared. + // False when an existing title equivalent row was adopted, or when a + // concurrent add or sync created it first. + BookCreated bool + // Author is the author the book was filed under. + Author *models.Author + // AuthorCreated is true when this call inserted the author row. + AuthorCreated bool +} + +// Refusals from addBookCore that carry no data. The AddBook handler maps +// each to the status and body it has always answered with (see +// addBookErrorResponses); other callers match them with errors.Is. +var ( + errAddBookForeignBookIDRequired = errors.New("add book: foreignBookId required") + errAddBookInvalidMediaType = errors.New("add book: invalid media type") + errAddBookAuthorMetadataUnavailable = errors.New("add book: author metadata unavailable") + errAddBookAuthorExists = errors.New("add book: author already exists") + errAddBookAuthorUnresolved = errors.New("add book: could not resolve author") + errAddBookCancelled = errors.New("add book: request cancelled") + errAddBookNotFound = errors.New("add book: book not found after author sync") + errAddBookHeldByAnotherUser = errors.New("add book: book is held by another user") +) + +// addBookErrorResponses holds the status and body for each sentinel above. +var addBookErrorResponses = []struct { + err error + status int + body string +}{ + {errAddBookForeignBookIDRequired, http.StatusBadRequest, "foreignBookId required"}, + {errAddBookInvalidMediaType, http.StatusBadRequest, "mediaType must be 'ebook', 'audiobook', or 'both'"}, + {errAddBookAuthorMetadataUnavailable, http.StatusUnprocessableEntity, "Author metadata unavailable for this result. Add the author manually first (Authors → Add Author by name), then try again."}, + {errAddBookAuthorExists, http.StatusConflict, "author already exists"}, + {errAddBookAuthorUnresolved, http.StatusInternalServerError, "could not resolve author"}, + {errAddBookCancelled, http.StatusGatewayTimeout, "request cancelled"}, + {errAddBookNotFound, http.StatusNotFound, "book not found after author sync — try again shortly"}, + {errAddBookHeldByAnotherUser, http.StatusConflict, "book is held by another user"}, +} + +// bookInLibraryError refuses to re-add a book the user can already see +// (#1227). Book is the existing row, which the handler returns in the 409. +type bookInLibraryError struct { + Book *models.Book +} + +func (e *bookInLibraryError) Error() string { return "add book: book already in library" } + +// primaryProviderUnavailableError refuses a fallback provider's record +// because the primary metadata provider did not answer (#2612). +type primaryProviderUnavailableError struct { + Primary string +} + +func (e *primaryProviderUnavailableError) Error() string { + return primaryProviderUnavailableMessage(e.Primary) +} + +// addBookLookupError wraps a failed book metadata lookup while resolving the +// author for a result that carried no author id. The handler answers 502 +// with the wrapped error's text. +type addBookLookupError struct { + Err error +} + +func (e *addBookLookupError) Error() string { return e.Err.Error() } +func (e *addBookLookupError) Unwrap() error { return e.Err } + +// writeAddBookError answers an addBookCore error with the status and body +// the AddBook handler returned before the core was split out. +func (h *AuthorHandler) writeAddBookError(w http.ResponseWriter, r *http.Request, err error) { + var inLibrary *bookInLibraryError + if errors.As(err, &inLibrary) { + h.writeExistingBookConflict(w, inLibrary.Book) + return + } + var unavailable *primaryProviderUnavailableError + if errors.As(err, &unavailable) { + writePrimaryProviderUnavailable(w, unavailable.Primary) + return + } + var lookup *addBookLookupError + if errors.As(err, &lookup) { + writeJSON(w, http.StatusBadGateway, map[string]string{"error": lookup.Err.Error()}) + return + } + for _, resp := range addBookErrorResponses { + if errors.Is(err, resp.err) { + writeJSON(w, resp.status, map[string]string{"error": resp.body}) + return + } + } + writeServerError(w, r, err) +} + +// addBookCore adds one book to the library: it resolves or creates the +// author, inserts the requested work, waits for the row, and marks it +// monitored. It is AddBook without the HTTP layer; see addBookParams for +// the inputs and the error types above for its refusals. +func (h *AuthorHandler) addBookCore(ctx context.Context, req addBookParams) (addBookResult, error) { + if req.ForeignBookID == "" { + return addBookResult{}, errAddBookForeignBookIDRequired + } + switch req.MediaType { + case "", models.MediaTypeEbook, models.MediaTypeAudiobook, models.MediaTypeBoth: + default: + return addBookResult{}, errAddBookInvalidMediaType + } + + // bookInserted and fallbackSynced record how the row came to exist, for + // addBookResult.BookCreated. + bookInserted := false + fallbackSynced := false + + // bookCreated flips true once the poll loop confirms the requested + // book is in the DB. The orphan-cleanup defer below reads it on + // AddBook return — when false (poll timeout, ctx cancel, etc.) the + // just-created author row is deleted iff it has zero books. Fixes + // issue #667 bug 3. + bookCreated := false + + // authorWasJustCreated tracks whether this request inserted the author + // row (vs. found it already present). Used by both the orphan-cleanup + // defer and the direct-insert block below — when the author was just + // created the async catalogue sync may take longer than the 15s poll + // budget for prolific authors, so we synchronously persist the requested + // book to guarantee it exists before the cleanup defer runs (#804). + authorWasJustCreated := false + + // 0. Refuse to re-add a book the user already owns (#1227). Before this + // check the handler reused the existing row and force-monitored it at + // step 3, so re-adding an owned book silently flipped it back to wanted. + // Runs before any author creation or upstream fetch so a conflict has no + // side effects. Scoped like the library list the user is looking at + // (ListScopeUserID plus NULL owners): a NULL owned row is in that list, + // and an admin under tenancy sees every row, so both are conflicts. A + // non admin's request never sees another user's copy here; the guard + // after the poll covers that case. + userID := auth.UserIDFromContext(ctx) + scopeID := auth.ListScopeUserID(ctx) + if existing, err := h.books.GetByForeignIDVisibleTo(ctx, req.ForeignBookID, scopeID); err != nil { + return addBookResult{}, err + } else if existing != nil { + return addBookResult{}, &bookInLibraryError{Book: existing} + } + + if req.ForeignAuthorID == "" { + resolved, outcome, err := h.resolveAuthorForBook(ctx, req.ForeignBookID) + if err != nil { + return addBookResult{}, &addBookLookupError{Err: err} + } + // The ISBN walk steps past a primary that timed out, so a fallback's + // record can win here only because the primary never answered. Adding + // it makes that provider the author's permanent link, and the same + // person is duplicated once the primary is back (#2117, #2612). Nothing + // has been written yet, so refuse and let a retry do the resolving. + if resolved != nil && !outcome.SafeToBind(resolved.Author.ForeignID) { + slog.Warn("AddBook: refusing to bind author to a fallback provider", + "foreignBookId", req.ForeignBookID, "primary", outcome.Primary, + "failed", outcome.FailureSummary(), "wouldHaveLinked", resolved.Author.ForeignID) + return addBookResult{}, &primaryProviderUnavailableError{Primary: outcome.Primary} + } + if resolved != nil { + // Rewrite the request so the existing fetch+poll flow targets the + // canonical provider's IDs. The user sees the canonical record (e.g. + // the OpenLibrary version) in their library; the original DNB record + // is dropped because bindery's author/book identity is single-source. + req.ForeignBookID = resolved.ForeignID + req.ForeignAuthorID = resolved.Author.ForeignID + if req.AuthorName == "" { + req.AuthorName = resolved.Author.Name + } + // The canonical id may itself already be in the library even though + // the id the client sent was not (#1227). + if existing, err := h.books.GetByForeignIDVisibleTo(ctx, req.ForeignBookID, scopeID); err != nil { + return addBookResult{}, err + } else if existing != nil { + return addBookResult{}, &bookInLibraryError{Book: existing} + } + } else if req.AuthorName != "" { + // ISBN-based resolution failed (e.g. Google Books: author name, no + // author ID, no ISBN). Resolve the author by NAME — prefer one already + // in the library so we reuse the user's existing author instead of + // duplicating it; otherwise adopt OpenLibrary's canonical record. Keep + // the chosen edition (req.ForeignBookID) — the other providers don't + // have this book. + if existing := h.findLibraryAuthorByName(ctx, req.AuthorName); existing != nil { + req.ForeignAuthorID = existing.ForeignID + } else if canonical, cErr := h.meta.ResolveCanonicalAuthor(ctx, req.AuthorName); cErr == nil && canonical != nil && outcome.SafeToBind(canonical.ForeignID) { + // The canonical lookup is OpenLibrary's, which is a fallback + // when another provider is primary, so the same guard applies. + req.ForeignAuthorID = canonical.ForeignID + } + } + if req.ForeignAuthorID == "" && outcome.PrimaryFailed { + // No provider placed the author, but the primary never answered, + // so "add the author manually" is the wrong advice: the primary + // may well know this book. + return addBookResult{}, &primaryProviderUnavailableError{Primary: outcome.Primary} + } + if req.ForeignAuthorID == "" { + return addBookResult{}, errAddBookAuthorMetadataUnavailable + } + } + + // 1. Find or create the author (unmonitored if new so we don't auto-want all books). + author, _ := h.authors.GetByForeignIDForUser(ctx, req.ForeignAuthorID, userID) + if author == nil { + author, _ = h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignAuthorID, userID) + } + if author == nil { + name := req.AuthorName + if name == "" { + name = req.ForeignAuthorID + } + fetched, err := h.meta.GetAuthor(ctx, req.ForeignAuthorID) + if err != nil || fetched == nil { + fetched = &models.Author{ + ForeignID: req.ForeignAuthorID, + Name: name, + SortName: sortName(name), + MetadataProvider: "openlibrary", + } + } + fetched.Monitored = false + def := models.DefaultMetadataProfileID + fetched.MetadataProfileID = &def + + // Dedupe path: if a canonical provider (OL / Hardcover / …) is being + // added for a SortName previously persisted as a synthetic DNB-only + // row, migrate that row in place rather than creating a duplicate. + // The synthetic row was created because the DNB record had only an + // author name (no GND link, no OL coverage). Now that a canonical + // identity exists, collapse the two onto a single primary key so + // the user keeps one author with all their books attached. + if !strings.HasPrefix(fetched.ForeignID, "dnb:") { + if existing, lookupErr := h.authors.GetByDNBSyntheticName(ctx, fetched.SortName, userID); lookupErr == nil && existing != nil { + if err := h.authors.UpgradeSyntheticDNB(ctx, existing.ForeignID, fetched); err != nil { + slog.Debug("AddBook: upgrade synthetic DNB author failed", "from", existing.ForeignID, "to", fetched.ForeignID, "error", err) + } else { + // Re-fetch the row by its new canonical ForeignID so subsequent + // steps see the upgraded record (ID preserved). + if upgraded, getErr := h.authors.GetByForeignIDForUser(ctx, fetched.ForeignID, userID); getErr == nil && upgraded != nil { + author = upgraded + } + } + } + } + + // CreateForUser may collide with a concurrent request inserting the + // same author; the UNIQUE-constraint branch below recovers by + // re-fetching the row. authorWasJustCreated stays false on the race + // path so the orphan-cleanup defer never rolls back somebody else's + // author row (issue #667). + if author == nil { + // Add-book creates the author as a side effect, so it never carries + // an explicit monitor choice — take the install-wide default (#1666). + db.ApplyAuthorMonitorDefaults(ctx, h.settings, fetched) + if err := h.authors.CreateForUser(ctx, fetched, userID); err != nil { + if !strings.Contains(err.Error(), "UNIQUE constraint failed") && !errors.Is(err, db.ErrAuthorIdentifierConflict) { + return addBookResult{}, err + } + // Race: another request created it between our check and insert. + author, _ = h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignAuthorID, userID) + if author == nil { + return addBookResult{}, errAddBookAuthorExists + } + } else { + author = fetched + authorWasJustCreated = true + // No speculative catalogue fetch here (#1816). Adding a book + // creates its author as a side effect; the user picked ONE + // title, and pulling that author's whole bibliography in behind + // it is the "my collection went from 75 books to over 500" + // report — the thing nobody expects because adding a film to + // Radarr does not import the director's filmography. + // + // Nothing downstream needs it: the direct insert below creates + // the picked book synchronously, which is what makes the poll + // succeed and what keeps the orphan-cleanup defer from + // rolling the author back. The narrow case where that insert + // cannot produce the row has its own single-work fallback, + // just past the direct-insert block. + } + } + // Defer the orphan cleanup so cancellation paths inside the poll + // loop also benefit. Runs only after a CreateForUser this request. + if authorWasJustCreated { + defer h.cleanupOrphanIfNoBooks(author, &bookCreated) + } + } + if author == nil { + return addBookResult{}, errAddBookAuthorUnresolved + } + + // 1b. Direct insert for the requested book. + // + // Originally added (#667) for DNB synthetic IDs, whose async sync returns + // zero books because DNB's public SRU has no author→works relationship. + // #804 widened this: for any author the request just created, the async + // catalogue sync can take longer than the 15 s poll budget (OpenLibrary + // took >32 s for a 175-work author in the bug report). When the poll + // times out, the orphan-cleanup defer deletes the author row — and the + // still-running goroutine then logs a FK-constraint failure for every + // book it tries to insert against the now-deleted author_id. + // + // Synchronously fetching and persisting the single requested record + // guarantees the poll succeeds on its first iteration AND that the + // cleanup defer sees a non-empty book list (so it keeps the author). + // The async sync still runs as a backfill for the rest of the catalogue; + // any UNIQUE collision against this row is silently tolerated. + // + // #1612 made the direct insert unconditional. When the author already + // EXISTED, AddBook used to skip it and rely entirely on a catalogue sync + // having created the row — but the sync can deterministically refuse a + // specific work (e.g. the work-level language sampled from the first few + // OpenLibrary editions falls outside the profile's allowed set, which is + // how heavily-translated works ended up permanently un-addable). Every + // attempt then polled 15 s for a row nothing would ever create and + // returned 404 "try again shortly" forever. An explicit add of one + // specific work is the strongest possible user signal and must not be + // vetoed by catalogue-sync heuristics; those heuristics still govern + // everything the user did NOT explicitly pick. + if existing, _ := h.books.GetByForeignID(ctx, req.ForeignBookID); existing == nil { + primary, err := h.meta.GetBook(ctx, req.ForeignBookID) + if err != nil { + slog.Warn("AddBook: direct fetch failed", + "foreignBookId", req.ForeignBookID, "error", err) + } else if primary != nil && h.directInsertTitleUsable(primary.Title, author.Name) { + primary.AuthorID = author.ID + // Tenancy (#1457): inherit the author's owner. Read off the author + // row rather than the request, because that row usually pre-exists + // this request (#1612) — the scoped lookup above is what makes it + // the correct owner either way. + primary.OwnerUserID = author.OwnerUserID + primary.Monitored = author.Monitored + if primary.Status == "" { + primary.Status = models.BookStatusWanted + } + // An explicit request choice wins over the provider's media type + // (#1397). Otherwise some providers (notably Google Books) don't + // set one; fall back to the global default so the row isn't + // created with an empty format (which would mis-route its + // indexer search). + if req.MediaType != "" { + primary.MediaType = req.MediaType + } else if primary.MediaType == "" { + primary.MediaType = h.resolveDefaultMediaType(ctx) + } + // Reuse a title-equivalent row under the same author instead of + // inserting a second one. The catalogue sync runs this same dedup + // (see the FindByAuthorAndDedupKey switch above), and skipping it + // here produced real duplicates: a Calibre-imported library holds + // the work under a `calibre:` foreign id, and OpenLibrary splits + // some works into separate ebook and audiobook Works that the sync + // merges into one media_type=both row — in both cases the + // requested foreign id has no row of its own, which is exactly the + // state that brings a user here. + match, ferr := h.books.FindByAuthorAndDedupKey(ctx, author.ID, primary.Title) + // A subtitle-collapsed dedup key (indexer.CanonicalDedupKey strips a + // ": subtitle" tail) merges every "Series: Volume" sibling onto one + // key. Adopting such a match would rebind the requested foreign id + // onto a *different* volume and — because adopt is a no-op when the + // row needs no field change — leave the poll below unable to find the + // requested id, returning 404 forever. When the requested work is a + // distinct volume of the matched row's series (same series, different + // sequence), skip the adopt and create a distinct row instead. + if ferr == nil && match != nil && !h.directInsertSeriesConflict(ctx, match.ID, primary.SeriesRefs) { + h.adoptDirectInsertMatch(ctx, match, primary, req.ForeignBookID) + } else if err := h.books.Create(ctx, primary); err != nil { + if !strings.Contains(err.Error(), "UNIQUE constraint failed") { + slog.Warn("AddBook: direct insert failed", + "foreignBookId", req.ForeignBookID, "error", err) + } + } else { + bookInserted = true + h.hydrateHardcoverEditions(ctx, primary, nil) + // Same post-create work every other creation path does + // (recommendations.go, series.go): check the library for a + // file we already have and link the book into its series. + // Without this the row is wanted-but-unchecked, so with + // searchOnAdd enabled Bindery re-downloads a book already on + // disk — the regression #940 and migration 026 exist to stop. + created := *primary + handleNewWantedBook(ctx, h.books, h.series, h.finder, created, author.Name) + } + } + } + + // 1c. Single-work fallback. The direct insert above covers the request in + // all but a couple of cases: the provider's book endpoint failed (#1612's + // OpenLibrary 502) or returned a record the title guard rejected, and the + // library has no row for the id either way. Ask the author endpoint for + // this ONE work instead — the same fetch the old speculative catalogue + // sync ran, restricted to the work the user actually picked, so the poll + // below can still succeed without the rest of the bibliography riding + // along (#1816). + if existing, _ := h.books.GetByForeignID(ctx, req.ForeignBookID); existing == nil { + // mediaType only fills a format the provider left blank, and step 3 + // below applies the request's explicit choice to whatever row the poll + // finds — so the default is the right value to pass here. It no longer + // decides whether the work is created at all: a single-work run is + // exempt from the strict media-type clamp (#1612). + fallbackSynced = true + h.fetchAuthorBooksAsync(author, catalogueSyncOptions{ + mediaType: h.resolveDefaultMediaType(ctx), + onlyForeignID: req.ForeignBookID, + }) + } + + // 2. Poll until the book appears (the single-work fallback, if it ran, + // creates it asynchronously). + deadline := time.Now().Add(15 * time.Second) + var book *models.Book + for { + b, _ := h.books.GetByForeignID(ctx, req.ForeignBookID) + if b != nil { + book = b + break + } + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return addBookResult{}, errAddBookCancelled + case <-time.After(500 * time.Millisecond): + } + } + + if book == nil { + return addBookResult{}, errAddBookNotFound + } + // The poll looks the row up globally because books.foreign_id is UNIQUE + // across users, so under tenancy it can return another user's copy: the + // conflict gate above never saw it (scoped to this user), the direct + // insert skipped it, and until #1227 the update below then flipped that + // user's book to monitored. Refuse instead, with no row in the body since + // the caller is not allowed to see it. bookCreated stays false so the + // orphan cleanup defer removes any author row this request created. + if book.OwnerUserID != 0 && userID != 0 && book.OwnerUserID != userID { + return addBookResult{}, errAddBookHeldByAnotherUser + } + bookCreated = true + + // 3. Mark the book monitored (wanted). An explicit media-type choice is + // applied here too — the poll may have found a row created by the async + // catalogue sync (or one already in the library) carrying the default. + // Re-evaluate status on a change so e.g. adding an already-imported ebook + // as 'both' flips it back to wanted for the missing format (#1148). + book.Monitored = true + if req.MediaType != "" && book.MediaType != req.MediaType { + book.MediaType = req.MediaType + reevaluateBookStatus(book) + } + if err := h.books.Update(ctx, book); err != nil { + return addBookResult{}, err + } + + // 3b. Say so when this add went past the strict media-type policy (#1759). + // + // The policy is a catalogue-population rule, not a veto on what the user + // may own: the direct insert above never consults it, and the single-work + // fallback is exempt by #1612's rule that "an explicit add of one specific + // work must not be vetoed by catalogue-sync heuristics". Both of those are + // deliberate, because silently refusing an explicit user action is the + // worse of the two failures. + // + // What was missing is that it happened invisibly, so a user who turned the + // setting on to stop un-grabbable rows appearing had no way to learn that + // their own add was the exception. The setting's help text now says the + // same thing, which is the half most people will actually see. + h.logStrictMediaTypeBypass(ctx, book) + + // 4. Optionally trigger an indexer search. Use the process-lifecycle + // context so the search goroutine is cancelled on shutdown rather than + // running against context.Background(). See #846. + if req.SearchOnAdd && h.searcher != nil { + go h.searcher.SearchAndGrabBook(indexer.WithSearchOrigin(h.bgCtx(), indexer.OriginAuthor), *book) // #nosec G118 -- intentional: search must outlive the request + } + + return addBookResult{ + Book: book, + BookCreated: bookInserted || fallbackSynced, + Author: author, + AuthorCreated: authorWasJustCreated, + }, nil +} diff --git a/internal/api/authors.go b/internal/api/authors.go index 01895ace5..86813892f 100644 --- a/internal/api/authors.go +++ b/internal/api/authors.go @@ -25,7 +25,6 @@ import ( "github.com/vavallee/bindery/internal/jobs" "github.com/vavallee/bindery/internal/metadata" "github.com/vavallee/bindery/internal/models" - "github.com/vavallee/bindery/internal/telemetry" "github.com/vavallee/bindery/internal/textutil" ) @@ -520,128 +519,29 @@ func (h *AuthorHandler) Create(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) return } - if req.ForeignID == "" || req.Name == "" { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "foreignAuthorId and authorName required"}) - return - } - monitorMode, monitorLatestCount, err := h.resolveCreateMonitorOptions(r.Context(), req.MonitorMode, req.MonitorLatestCount) - monitorNewItems := models.DefaultAuthorMonitorNewItems - if req.MonitorNewItems != nil { - v := strings.TrimSpace(*req.MonitorNewItems) - if !models.IsAuthorMonitorNewItemsValid(v) { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid monitorNewItems"}) - return - } - monitorNewItems = v - } - if err != nil { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) - return - } - - // Check if already exists — use user-scoped lookup so this agrees with the - // author list, which filters by owner_user_id. A global GetByForeignID - // would block re-creation of authors orphaned under a different user ID. - userID := auth.UserIDFromContext(r.Context()) - existing, _ := h.authors.GetByAnyForeignIDForUser(r.Context(), req.ForeignID, userID) - if existing != nil { - h.writeCanonicalAuthorConflict(w, existing, "author already exists") - return - } - - author, err := h.fetchAuthorForCreate(r.Context(), req.ForeignID, req.Name) + res, err := h.createAuthorCore(r.Context(), createAuthorParams{ + ForeignID: req.ForeignID, + Name: req.Name, + QualityProfileID: req.QualityProfileID, + MetadataProfileID: req.MetadataProfileID, + RootFolderID: req.RootFolderID, + AudiobookRootFolderID: req.AudiobookRootFolderID, + Monitored: req.Monitored, + MonitorMode: req.MonitorMode, + MonitorLatestCount: req.MonitorLatestCount, + MonitorNewItems: req.MonitorNewItems, + SearchOnAdd: req.SearchOnAdd, + MediaType: req.MediaType, + }) if err != nil { - writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()}) + h.writeCreateAuthorError(w, r, err) return } - if author.ForeignID != "" { - if existing, _ := h.authors.GetByAnyForeignIDForUser(r.Context(), author.ForeignID, userID); existing != nil { - h.writeCanonicalAuthorConflict(w, existing, "author already exists") - return - } - } - if canonical, ambiguous, err := h.findCanonicalAuthorMatch(r.Context(), req.Name, author.Name); err != nil { - writeServerError(w, r, err) - return - } else if ambiguous { - writeJSON(w, http.StatusConflict, map[string]string{"error": "author name resolves ambiguously — merge manually"}) - return - } else if canonical != nil { - if canRelinkAuthorToUpstream(canonical) { - if err := h.relinkExistingAuthorToUpstream(r.Context(), canonical, author, req.Name, req.Monitored, monitorMode, monitorLatestCount, monitorNewItems, req.QualityProfileID, req.MetadataProfileID, req.RootFolderID, req.AudiobookRootFolderID); err != nil { - if isAuthorIdentityConflict(err) { - writeJSON(w, http.StatusConflict, map[string]string{"error": "upstream author already exists locally"}) - return - } - writeServerError(w, r, err) - return - } - mediaType := req.MediaType - if mediaType == "" { - mediaType = h.resolveDefaultMediaType(r.Context()) - } - // Finish mutating `canonical` (description clean-up) BEFORE spawning - // the async catalogue+profile refresh. fetchAuthorBooksAsync snapshots - // the author at spawn time and the goroutine now reads/writes profile - // fields (Description, ImageURL, ...); cleaning afterwards would race - // the snapshot read against this write (see fetchAuthorBooksAsync). - cleanAuthorDescription(canonical) - h.stampProviderMismatch(canonical) - h.fetchAuthorBooksAsync(canonical, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) - writeJSON(w, http.StatusOK, canonical) - return - } - h.writeCanonicalAuthorConflict(w, canonical, "author name already resolves to an existing author — confirm merge") + if !res.Created { + writeJSON(w, http.StatusOK, res.Author) return } - applyAuthorCreateOptions(author, req.Monitored, monitorMode, monitorLatestCount, req.QualityProfileID, req.MetadataProfileID, req.RootFolderID, req.AudiobookRootFolderID) - author.MonitorNewItems = monitorNewItems - - if err := h.authors.CreateForUser(r.Context(), author, auth.UserIDFromContext(r.Context())); err != nil { - slog.Error("create author failed", "foreign_id", req.ForeignID, "error", err) - if strings.Contains(err.Error(), "UNIQUE constraint failed") || errors.Is(err, db.ErrAuthorIdentifierConflict) { - if existing, _ := h.authors.GetByAnyForeignIDForUser(r.Context(), req.ForeignID, userID); existing != nil { - h.writeCanonicalAuthorConflict(w, existing, "author already exists") - return - } - if existing, _ := h.authors.GetByAnyForeignIDForUser(r.Context(), author.ForeignID, userID); existing != nil { - h.writeCanonicalAuthorConflict(w, existing, "author already exists") - return - } - writeJSON(w, http.StatusConflict, map[string]string{"error": "author already exists"}) - return - } - writeServerError(w, r, err) - return - } - h.recordAuthorCreateAlias(r.Context(), author, req.Name) - - // Persist any OL alternate names as alias rows so non-latin primary names - // (e.g. "村上春樹") get their latin-script alternates ("Haruki Murakami") - // indexed for release-name matching. - h.saveAlternateNames(r.Context(), author) - - // Resolve effective media type for books created under this author: - // explicit request value wins, else the global default.media_type - // setting, else ebook (backwards compat). - mediaType := req.MediaType - if mediaType == "" { - mediaType = h.resolveDefaultMediaType(r.Context()) - } - - // Clean the description BEFORE spawning the async refresh: the goroutine - // snapshots `author` and now reads/writes its profile fields (Description, - // ImageURL, ...). Cleaning after the spawn would race the snapshot read - // against this write (see fetchAuthorBooksAsync). - cleanAuthorDescription(author) - h.stampProviderMismatch(author) - - // Fetch and store books for this author. Always populate the catalogue; - // pass searchOnAdd so FetchAuthorBooks knows whether to also queue grabs. - h.fetchAuthorBooksAsync(author, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) - - telemetry.MarkFirst(r.Context(), h.settings, telemetry.SettingFirstAuthorAt) - writeJSON(w, http.StatusCreated, author) + writeJSON(w, http.StatusCreated, res.Author) } func cleanAuthorDescription(author *models.Author) { @@ -3085,402 +2985,18 @@ func (h *AuthorHandler) AddBook(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) return } - if req.ForeignBookID == "" { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "foreignBookId required"}) - return - } - switch req.MediaType { - case "", models.MediaTypeEbook, models.MediaTypeAudiobook, models.MediaTypeBoth: - default: - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "mediaType must be 'ebook', 'audiobook', or 'both'"}) - return - } - - ctx := r.Context() - - // bookCreated flips true once the poll loop confirms the requested - // book is in the DB. The orphan-cleanup defer below reads it on - // AddBook return — when false (poll timeout, ctx cancel, etc.) the - // just-created author row is deleted iff it has zero books. Fixes - // issue #667 bug 3. - bookCreated := false - - // authorWasJustCreated tracks whether this request inserted the author - // row (vs. found it already present). Used by both the orphan-cleanup - // defer and the direct-insert block below — when the author was just - // created the async catalogue sync may take longer than the 15s poll - // budget for prolific authors, so we synchronously persist the requested - // book to guarantee it exists before the cleanup defer runs (#804). - authorWasJustCreated := false - - // 0. Refuse to re-add a book the user already owns (#1227). Before this - // check the handler reused the existing row and force-monitored it at - // step 3, so re-adding an owned book silently flipped it back to wanted. - // Runs before any author creation or upstream fetch so a conflict has no - // side effects. Scoped like the library list the user is looking at - // (ListScopeUserID plus NULL owners): a NULL owned row is in that list, - // and an admin under tenancy sees every row, so both are conflicts. A - // non admin's request never sees another user's copy here; the guard - // after the poll covers that case. - userID := auth.UserIDFromContext(ctx) - scopeID := auth.ListScopeUserID(ctx) - if existing, err := h.books.GetByForeignIDVisibleTo(ctx, req.ForeignBookID, scopeID); err != nil { - writeServerError(w, r, err) - return - } else if existing != nil { - h.writeExistingBookConflict(w, existing) - return - } - - if req.ForeignAuthorID == "" { - resolved, outcome, err := h.resolveAuthorForBook(ctx, req.ForeignBookID) - if err != nil { - writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()}) - return - } - // The ISBN walk steps past a primary that timed out, so a fallback's - // record can win here only because the primary never answered. Adding - // it makes that provider the author's permanent link, and the same - // person is duplicated once the primary is back (#2117, #2612). Nothing - // has been written yet, so refuse and let a retry do the resolving. - if resolved != nil && !outcome.SafeToBind(resolved.Author.ForeignID) { - slog.Warn("AddBook: refusing to bind author to a fallback provider", - "foreignBookId", req.ForeignBookID, "primary", outcome.Primary, - "failed", outcome.FailureSummary(), "wouldHaveLinked", resolved.Author.ForeignID) - writePrimaryProviderUnavailable(w, outcome.Primary) - return - } - if resolved != nil { - // Rewrite the request so the existing fetch+poll flow targets the - // canonical provider's IDs. The user sees the canonical record (e.g. - // the OpenLibrary version) in their library; the original DNB record - // is dropped because bindery's author/book identity is single-source. - req.ForeignBookID = resolved.ForeignID - req.ForeignAuthorID = resolved.Author.ForeignID - if req.AuthorName == "" { - req.AuthorName = resolved.Author.Name - } - // The canonical id may itself already be in the library even though - // the id the client sent was not (#1227). - if existing, err := h.books.GetByForeignIDVisibleTo(ctx, req.ForeignBookID, scopeID); err != nil { - writeServerError(w, r, err) - return - } else if existing != nil { - h.writeExistingBookConflict(w, existing) - return - } - } else if req.AuthorName != "" { - // ISBN-based resolution failed (e.g. Google Books: author name, no - // author ID, no ISBN). Resolve the author by NAME — prefer one already - // in the library so we reuse the user's existing author instead of - // duplicating it; otherwise adopt OpenLibrary's canonical record. Keep - // the chosen edition (req.ForeignBookID) — the other providers don't - // have this book. - if existing := h.findLibraryAuthorByName(ctx, req.AuthorName); existing != nil { - req.ForeignAuthorID = existing.ForeignID - } else if canonical, cErr := h.meta.ResolveCanonicalAuthor(ctx, req.AuthorName); cErr == nil && canonical != nil && outcome.SafeToBind(canonical.ForeignID) { - // The canonical lookup is OpenLibrary's, which is a fallback - // when another provider is primary, so the same guard applies. - req.ForeignAuthorID = canonical.ForeignID - } - } - if req.ForeignAuthorID == "" && outcome.PrimaryFailed { - // No provider placed the author, but the primary never answered, - // so "add the author manually" is the wrong advice: the primary - // may well know this book. - writePrimaryProviderUnavailable(w, outcome.Primary) - return - } - if req.ForeignAuthorID == "" { - writeJSON(w, http.StatusUnprocessableEntity, map[string]string{ - "error": "Author metadata unavailable for this result. Add the author manually first (Authors → Add Author by name), then try again.", - }) - return - } - } - - // 1. Find or create the author (unmonitored if new so we don't auto-want all books). - author, _ := h.authors.GetByForeignIDForUser(ctx, req.ForeignAuthorID, userID) - if author == nil { - author, _ = h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignAuthorID, userID) - } - if author == nil { - name := req.AuthorName - if name == "" { - name = req.ForeignAuthorID - } - fetched, err := h.meta.GetAuthor(ctx, req.ForeignAuthorID) - if err != nil || fetched == nil { - fetched = &models.Author{ - ForeignID: req.ForeignAuthorID, - Name: name, - SortName: sortName(name), - MetadataProvider: "openlibrary", - } - } - fetched.Monitored = false - def := models.DefaultMetadataProfileID - fetched.MetadataProfileID = &def - - // Dedupe path: if a canonical provider (OL / Hardcover / …) is being - // added for a SortName previously persisted as a synthetic DNB-only - // row, migrate that row in place rather than creating a duplicate. - // The synthetic row was created because the DNB record had only an - // author name (no GND link, no OL coverage). Now that a canonical - // identity exists, collapse the two onto a single primary key so - // the user keeps one author with all their books attached. - if !strings.HasPrefix(fetched.ForeignID, "dnb:") { - if existing, lookupErr := h.authors.GetByDNBSyntheticName(ctx, fetched.SortName, userID); lookupErr == nil && existing != nil { - if err := h.authors.UpgradeSyntheticDNB(ctx, existing.ForeignID, fetched); err != nil { - slog.Debug("AddBook: upgrade synthetic DNB author failed", "from", existing.ForeignID, "to", fetched.ForeignID, "error", err) - } else { - // Re-fetch the row by its new canonical ForeignID so subsequent - // steps see the upgraded record (ID preserved). - if upgraded, getErr := h.authors.GetByForeignIDForUser(ctx, fetched.ForeignID, userID); getErr == nil && upgraded != nil { - author = upgraded - } - } - } - } - - // CreateForUser may collide with a concurrent request inserting the - // same author; the UNIQUE-constraint branch below recovers by - // re-fetching the row. authorWasJustCreated stays false on the race - // path so the orphan-cleanup defer never rolls back somebody else's - // author row (issue #667). - if author == nil { - // Add-book creates the author as a side effect, so it never carries - // an explicit monitor choice — take the install-wide default (#1666). - db.ApplyAuthorMonitorDefaults(ctx, h.settings, fetched) - if err := h.authors.CreateForUser(ctx, fetched, userID); err != nil { - if !strings.Contains(err.Error(), "UNIQUE constraint failed") && !errors.Is(err, db.ErrAuthorIdentifierConflict) { - writeServerError(w, r, err) - return - } - // Race: another request created it between our check and insert. - author, _ = h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignAuthorID, userID) - if author == nil { - writeJSON(w, http.StatusConflict, map[string]string{"error": "author already exists"}) - return - } - } else { - author = fetched - authorWasJustCreated = true - // No speculative catalogue fetch here (#1816). Adding a book - // creates its author as a side effect; the user picked ONE - // title, and pulling that author's whole bibliography in behind - // it is the "my collection went from 75 books to over 500" - // report — the thing nobody expects because adding a film to - // Radarr does not import the director's filmography. - // - // Nothing downstream needs it: the direct insert below creates - // the picked book synchronously, which is what makes the poll - // succeed and what keeps the orphan-cleanup defer from - // rolling the author back. The narrow case where that insert - // cannot produce the row has its own single-work fallback, - // just past the direct-insert block. - } - } - // Defer the orphan cleanup so cancellation paths inside the poll - // loop also benefit. Runs only after a CreateForUser this request. - if authorWasJustCreated { - defer h.cleanupOrphanIfNoBooks(author, &bookCreated) - } - } - if author == nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not resolve author"}) - return - } - - // 1b. Direct insert for the requested book. - // - // Originally added (#667) for DNB synthetic IDs, whose async sync returns - // zero books because DNB's public SRU has no author→works relationship. - // #804 widened this: for any author the request just created, the async - // catalogue sync can take longer than the 15 s poll budget (OpenLibrary - // took >32 s for a 175-work author in the bug report). When the poll - // times out, the orphan-cleanup defer deletes the author row — and the - // still-running goroutine then logs a FK-constraint failure for every - // book it tries to insert against the now-deleted author_id. - // - // Synchronously fetching and persisting the single requested record - // guarantees the poll succeeds on its first iteration AND that the - // cleanup defer sees a non-empty book list (so it keeps the author). - // The async sync still runs as a backfill for the rest of the catalogue; - // any UNIQUE collision against this row is silently tolerated. - // - // #1612 made the direct insert unconditional. When the author already - // EXISTED, AddBook used to skip it and rely entirely on a catalogue sync - // having created the row — but the sync can deterministically refuse a - // specific work (e.g. the work-level language sampled from the first few - // OpenLibrary editions falls outside the profile's allowed set, which is - // how heavily-translated works ended up permanently un-addable). Every - // attempt then polled 15 s for a row nothing would ever create and - // returned 404 "try again shortly" forever. An explicit add of one - // specific work is the strongest possible user signal and must not be - // vetoed by catalogue-sync heuristics; those heuristics still govern - // everything the user did NOT explicitly pick. - if existing, _ := h.books.GetByForeignID(ctx, req.ForeignBookID); existing == nil { - primary, err := h.meta.GetBook(ctx, req.ForeignBookID) - if err != nil { - slog.Warn("AddBook: direct fetch failed", - "foreignBookId", req.ForeignBookID, "error", err) - } else if primary != nil && h.directInsertTitleUsable(primary.Title, author.Name) { - primary.AuthorID = author.ID - // Tenancy (#1457): inherit the author's owner. Read off the author - // row rather than the request, because that row usually pre-exists - // this request (#1612) — the scoped lookup above is what makes it - // the correct owner either way. - primary.OwnerUserID = author.OwnerUserID - primary.Monitored = author.Monitored - if primary.Status == "" { - primary.Status = models.BookStatusWanted - } - // An explicit request choice wins over the provider's media type - // (#1397). Otherwise some providers (notably Google Books) don't - // set one; fall back to the global default so the row isn't - // created with an empty format (which would mis-route its - // indexer search). - if req.MediaType != "" { - primary.MediaType = req.MediaType - } else if primary.MediaType == "" { - primary.MediaType = h.resolveDefaultMediaType(ctx) - } - // Reuse a title-equivalent row under the same author instead of - // inserting a second one. The catalogue sync runs this same dedup - // (see the FindByAuthorAndDedupKey switch above), and skipping it - // here produced real duplicates: a Calibre-imported library holds - // the work under a `calibre:` foreign id, and OpenLibrary splits - // some works into separate ebook and audiobook Works that the sync - // merges into one media_type=both row — in both cases the - // requested foreign id has no row of its own, which is exactly the - // state that brings a user here. - match, ferr := h.books.FindByAuthorAndDedupKey(ctx, author.ID, primary.Title) - // A subtitle-collapsed dedup key (indexer.CanonicalDedupKey strips a - // ": subtitle" tail) merges every "Series: Volume" sibling onto one - // key. Adopting such a match would rebind the requested foreign id - // onto a *different* volume and — because adopt is a no-op when the - // row needs no field change — leave the poll below unable to find the - // requested id, returning 404 forever. When the requested work is a - // distinct volume of the matched row's series (same series, different - // sequence), skip the adopt and create a distinct row instead. - if ferr == nil && match != nil && !h.directInsertSeriesConflict(ctx, match.ID, primary.SeriesRefs) { - h.adoptDirectInsertMatch(ctx, match, primary, req.ForeignBookID) - } else if err := h.books.Create(ctx, primary); err != nil { - if !strings.Contains(err.Error(), "UNIQUE constraint failed") { - slog.Warn("AddBook: direct insert failed", - "foreignBookId", req.ForeignBookID, "error", err) - } - } else { - h.hydrateHardcoverEditions(ctx, primary, nil) - // Same post-create work every other creation path does - // (recommendations.go, series.go): check the library for a - // file we already have and link the book into its series. - // Without this the row is wanted-but-unchecked, so with - // searchOnAdd enabled Bindery re-downloads a book already on - // disk — the regression #940 and migration 026 exist to stop. - created := *primary - handleNewWantedBook(ctx, h.books, h.series, h.finder, created, author.Name) - } - } - } - - // 1c. Single-work fallback. The direct insert above covers the request in - // all but a couple of cases: the provider's book endpoint failed (#1612's - // OpenLibrary 502) or returned a record the title guard rejected, and the - // library has no row for the id either way. Ask the author endpoint for - // this ONE work instead — the same fetch the old speculative catalogue - // sync ran, restricted to the work the user actually picked, so the poll - // below can still succeed without the rest of the bibliography riding - // along (#1816). - if existing, _ := h.books.GetByForeignID(ctx, req.ForeignBookID); existing == nil { - // mediaType only fills a format the provider left blank, and step 3 - // below applies the request's explicit choice to whatever row the poll - // finds — so the default is the right value to pass here. It no longer - // decides whether the work is created at all: a single-work run is - // exempt from the strict media-type clamp (#1612). - h.fetchAuthorBooksAsync(author, catalogueSyncOptions{ - mediaType: h.resolveDefaultMediaType(ctx), - onlyForeignID: req.ForeignBookID, - }) - } - - // 2. Poll until the book appears (the single-work fallback, if it ran, - // creates it asynchronously). - deadline := time.Now().Add(15 * time.Second) - var book *models.Book - for { - b, _ := h.books.GetByForeignID(ctx, req.ForeignBookID) - if b != nil { - book = b - break - } - if time.Now().After(deadline) { - break - } - select { - case <-ctx.Done(): - writeJSON(w, http.StatusGatewayTimeout, map[string]string{"error": "request cancelled"}) - return - case <-time.After(500 * time.Millisecond): - } - } - - if book == nil { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "book not found after author sync — try again shortly"}) - return - } - // The poll looks the row up globally because books.foreign_id is UNIQUE - // across users, so under tenancy it can return another user's copy: the - // conflict gate above never saw it (scoped to this user), the direct - // insert skipped it, and until #1227 the update below then flipped that - // user's book to monitored. Refuse instead, with no row in the body since - // the caller is not allowed to see it. bookCreated stays false so the - // orphan cleanup defer removes any author row this request created. - if book.OwnerUserID != 0 && userID != 0 && book.OwnerUserID != userID { - writeJSON(w, http.StatusConflict, map[string]string{"error": "book is held by another user"}) - return - } - bookCreated = true - - // 3. Mark the book monitored (wanted). An explicit media-type choice is - // applied here too — the poll may have found a row created by the async - // catalogue sync (or one already in the library) carrying the default. - // Re-evaluate status on a change so e.g. adding an already-imported ebook - // as 'both' flips it back to wanted for the missing format (#1148). - book.Monitored = true - if req.MediaType != "" && book.MediaType != req.MediaType { - book.MediaType = req.MediaType - reevaluateBookStatus(book) - } - if err := h.books.Update(ctx, book); err != nil { - writeServerError(w, r, err) + res, err := h.addBookCore(r.Context(), addBookParams{ + ForeignBookID: req.ForeignBookID, + ForeignAuthorID: req.ForeignAuthorID, + AuthorName: req.AuthorName, + SearchOnAdd: req.SearchOnAdd, + MediaType: req.MediaType, + }) + if err != nil { + h.writeAddBookError(w, r, err) return } - - // 3b. Say so when this add went past the strict media-type policy (#1759). - // - // The policy is a catalogue-population rule, not a veto on what the user - // may own: the direct insert above never consults it, and the single-work - // fallback is exempt by #1612's rule that "an explicit add of one specific - // work must not be vetoed by catalogue-sync heuristics". Both of those are - // deliberate, because silently refusing an explicit user action is the - // worse of the two failures. - // - // What was missing is that it happened invisibly, so a user who turned the - // setting on to stop un-grabbable rows appearing had no way to learn that - // their own add was the exception. The setting's help text now says the - // same thing, which is the half most people will actually see. - h.logStrictMediaTypeBypass(ctx, book) - - // 4. Optionally trigger an indexer search. Use the process-lifecycle - // context so the search goroutine is cancelled on shutdown rather than - // running against context.Background(). See #846. - if req.SearchOnAdd && h.searcher != nil { - go h.searcher.SearchAndGrabBook(indexer.WithSearchOrigin(h.bgCtx(), indexer.OriginAuthor), *book) // #nosec G118 -- intentional: search must outlive the request - } - - writeJSON(w, http.StatusCreated, book) + writeJSON(w, http.StatusCreated, res.Book) } // logStrictMediaTypeBypass records an explicit add that the strict media-type diff --git a/internal/api/create_author_core.go b/internal/api/create_author_core.go new file mode 100644 index 000000000..e59bb2608 --- /dev/null +++ b/internal/api/create_author_core.go @@ -0,0 +1,229 @@ +package api + +import ( + "context" + "errors" + "log/slog" + "net/http" + "strings" + + "github.com/vavallee/bindery/internal/auth" + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/models" + "github.com/vavallee/bindery/internal/telemetry" +) + +// createAuthorParams is everything an Add Author needs, independent of HTTP. +// The Create handler fills it from the request body; request approval fills +// it itself. Ownership is not a field: the core reads the acting user from +// ctx, exactly as the handler always has. +type createAuthorParams struct { + ForeignID string + Name string + QualityProfileID *int64 + MetadataProfileID *int64 + RootFolderID *int64 + AudiobookRootFolderID *int64 + Monitored bool + MonitorMode *string + MonitorLatestCount *int + // Settable at add time as of the Add Author redesign: before this it + // could only be changed on the author afterwards, so "catalogue this + // author once and never let a refresh grow it" was not expressible + // when adding. Same validation as Update. + MonitorNewItems *string + SearchOnAdd bool + MediaType string +} + +// createAuthorResult is what an Add Author produced. +type createAuthorResult struct { + // Author is the created author, or the existing author relinked to the + // requested upstream record. + Author *models.Author + // Created is true when a new author row was inserted (the handler's 201) + // and false when an existing author was relinked instead (its 200). + Created bool +} + +// Refusals from createAuthorCore that carry no data. The Create handler maps +// each to a 400 with the body it has always answered with. +var ( + errCreateAuthorFieldsRequired = errors.New("create author: foreignAuthorId and authorName required") + errCreateAuthorInvalidMonitorNewItems = errors.New("create author: invalid monitorNewItems") +) + +var createAuthorErrorResponses = []struct { + err error + body string +}{ + {errCreateAuthorFieldsRequired, "foreignAuthorId and authorName required"}, + {errCreateAuthorInvalidMonitorNewItems, "invalid monitorNewItems"}, +} + +// createAuthorOptionError is an invalid monitorMode or monitorLatestCount. +// The handler answers 400 with the wrapped error's text. +type createAuthorOptionError struct { + Err error +} + +func (e *createAuthorOptionError) Error() string { return e.Err.Error() } +func (e *createAuthorOptionError) Unwrap() error { return e.Err } + +// createAuthorLookupError is a failed upstream author fetch. The handler +// answers 502 with the wrapped error's text. +type createAuthorLookupError struct { + Err error +} + +func (e *createAuthorLookupError) Error() string { return e.Err.Error() } +func (e *createAuthorLookupError) Unwrap() error { return e.Err } + +// authorConflictError is a 409: the author already exists, or its name +// resolves to one that does. Canonical, when set, is the existing author the +// handler returns so the client can offer to open it. +type authorConflictError struct { + Canonical *models.Author + Message string +} + +func (e *authorConflictError) Error() string { return "create author: " + e.Message } + +// writeCreateAuthorError answers a createAuthorCore error with the status +// and body the Create handler returned before the core was split out. +func (h *AuthorHandler) writeCreateAuthorError(w http.ResponseWriter, r *http.Request, err error) { + var conflict *authorConflictError + if errors.As(err, &conflict) { + h.writeCanonicalAuthorConflict(w, conflict.Canonical, conflict.Message) + return + } + var option *createAuthorOptionError + if errors.As(err, &option) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": option.Err.Error()}) + return + } + var lookup *createAuthorLookupError + if errors.As(err, &lookup) { + writeJSON(w, http.StatusBadGateway, map[string]string{"error": lookup.Err.Error()}) + return + } + for _, resp := range createAuthorErrorResponses { + if errors.Is(err, resp.err) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": resp.body}) + return + } + } + writeServerError(w, r, err) +} + +// createAuthorCore adds one author and starts its catalogue sync, or relinks +// an existing local author to the requested upstream record. It is Create +// without the HTTP layer. +func (h *AuthorHandler) createAuthorCore(ctx context.Context, req createAuthorParams) (createAuthorResult, error) { + if req.ForeignID == "" || req.Name == "" { + return createAuthorResult{}, errCreateAuthorFieldsRequired + } + monitorMode, monitorLatestCount, err := h.resolveCreateMonitorOptions(ctx, req.MonitorMode, req.MonitorLatestCount) + monitorNewItems := models.DefaultAuthorMonitorNewItems + if req.MonitorNewItems != nil { + v := strings.TrimSpace(*req.MonitorNewItems) + if !models.IsAuthorMonitorNewItemsValid(v) { + return createAuthorResult{}, errCreateAuthorInvalidMonitorNewItems + } + monitorNewItems = v + } + if err != nil { + return createAuthorResult{}, &createAuthorOptionError{Err: err} + } + + // Check if already exists — use user-scoped lookup so this agrees with the + // author list, which filters by owner_user_id. A global GetByForeignID + // would block re-creation of authors orphaned under a different user ID. + userID := auth.UserIDFromContext(ctx) + existing, _ := h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignID, userID) + if existing != nil { + return createAuthorResult{}, &authorConflictError{Canonical: existing, Message: "author already exists"} + } + + author, err := h.fetchAuthorForCreate(ctx, req.ForeignID, req.Name) + if err != nil { + return createAuthorResult{}, &createAuthorLookupError{Err: err} + } + if author.ForeignID != "" { + if existing, _ := h.authors.GetByAnyForeignIDForUser(ctx, author.ForeignID, userID); existing != nil { + return createAuthorResult{}, &authorConflictError{Canonical: existing, Message: "author already exists"} + } + } + if canonical, ambiguous, err := h.findCanonicalAuthorMatch(ctx, req.Name, author.Name); err != nil { + return createAuthorResult{}, err + } else if ambiguous { + return createAuthorResult{}, &authorConflictError{Message: "author name resolves ambiguously — merge manually"} + } else if canonical != nil { + if canRelinkAuthorToUpstream(canonical) { + if err := h.relinkExistingAuthorToUpstream(ctx, canonical, author, req.Name, req.Monitored, monitorMode, monitorLatestCount, monitorNewItems, req.QualityProfileID, req.MetadataProfileID, req.RootFolderID, req.AudiobookRootFolderID); err != nil { + if isAuthorIdentityConflict(err) { + return createAuthorResult{}, &authorConflictError{Message: "upstream author already exists locally"} + } + return createAuthorResult{}, err + } + mediaType := req.MediaType + if mediaType == "" { + mediaType = h.resolveDefaultMediaType(ctx) + } + // Finish mutating `canonical` (description clean-up) BEFORE spawning + // the async catalogue+profile refresh. fetchAuthorBooksAsync snapshots + // the author at spawn time and the goroutine now reads/writes profile + // fields (Description, ImageURL, ...); cleaning afterwards would race + // the snapshot read against this write (see fetchAuthorBooksAsync). + cleanAuthorDescription(canonical) + h.stampProviderMismatch(canonical) + h.fetchAuthorBooksAsync(canonical, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) + return createAuthorResult{Author: canonical}, nil + } + return createAuthorResult{}, &authorConflictError{Canonical: canonical, Message: "author name already resolves to an existing author — confirm merge"} + } + applyAuthorCreateOptions(author, req.Monitored, monitorMode, monitorLatestCount, req.QualityProfileID, req.MetadataProfileID, req.RootFolderID, req.AudiobookRootFolderID) + author.MonitorNewItems = monitorNewItems + + if err := h.authors.CreateForUser(ctx, author, auth.UserIDFromContext(ctx)); err != nil { + slog.Error("create author failed", "foreign_id", req.ForeignID, "error", err) + if strings.Contains(err.Error(), "UNIQUE constraint failed") || errors.Is(err, db.ErrAuthorIdentifierConflict) { + if existing, _ := h.authors.GetByAnyForeignIDForUser(ctx, req.ForeignID, userID); existing != nil { + return createAuthorResult{}, &authorConflictError{Canonical: existing, Message: "author already exists"} + } + if existing, _ := h.authors.GetByAnyForeignIDForUser(ctx, author.ForeignID, userID); existing != nil { + return createAuthorResult{}, &authorConflictError{Canonical: existing, Message: "author already exists"} + } + return createAuthorResult{}, &authorConflictError{Message: "author already exists"} + } + return createAuthorResult{}, err + } + h.recordAuthorCreateAlias(ctx, author, req.Name) + + // Persist any OL alternate names as alias rows so non-latin primary names + // (e.g. "村上春樹") get their latin-script alternates ("Haruki Murakami") + // indexed for release-name matching. + h.saveAlternateNames(ctx, author) + + // Resolve effective media type for books created under this author: + // explicit request value wins, else the global default.media_type + // setting, else ebook (backwards compat). + mediaType := req.MediaType + if mediaType == "" { + mediaType = h.resolveDefaultMediaType(ctx) + } + + // Clean the description BEFORE spawning the async refresh: the goroutine + // snapshots `author` and now reads/writes its profile fields (Description, + // ImageURL, ...). Cleaning after the spawn would race the snapshot read + // against this write (see fetchAuthorBooksAsync). + cleanAuthorDescription(author) + h.stampProviderMismatch(author) + + // Fetch and store books for this author. Always populate the catalogue; + // pass searchOnAdd so FetchAuthorBooks knows whether to also queue grabs. + h.fetchAuthorBooksAsync(author, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) + + telemetry.MarkFirst(ctx, h.settings, telemetry.SettingFirstAuthorAt) + return createAuthorResult{Author: author, Created: true}, nil +} From 58292deaa10948f0e23af2cde6ecbda867f046d9 Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 01:15:16 -0300 Subject: [PATCH 02/18] refactor(api): add Monitored and SkipCatalogueSync overrides to the add cores Library adoption creates books unmonitored and must not pull in a catalogue behind them; request approval needs the same control over the author sync. Both overrides default to what the handlers already do, and the handlers never set them. addBookParams.Monitored nil keeps the forced monitored flag. addBookParams.SkipCatalogueSync skips the single work fallback sync and looks for the row once instead of polling, since nothing the core started could still create it. createAuthorParams.SkipCatalogueSync skips the catalogue sync on both the create and the relink path. New tests call the cores directly: default params match the handler, Monitored=false stores an unmonitored book, SkipCatalogueSync makes no author works call (with a control case that does), and the handler's error table matches the literal bodies. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- internal/api/add_book_core.go | 16 +- internal/api/add_book_core_test.go | 360 +++++++++++++++++++++++++++++ internal/api/create_author_core.go | 11 +- 3 files changed, 384 insertions(+), 3 deletions(-) create mode 100644 internal/api/add_book_core_test.go diff --git a/internal/api/add_book_core.go b/internal/api/add_book_core.go index a899c8ca7..ca853d85a 100644 --- a/internal/api/add_book_core.go +++ b/internal/api/add_book_core.go @@ -27,6 +27,14 @@ type addBookParams struct { // book (#1397). Empty keeps the provider's media type, falling back to // the default.media_type setting. MediaType string + // Monitored overrides the monitored flag step 3 sets on the book. Nil + // keeps the handler's behaviour, which always marks the book monitored. + Monitored *bool + // SkipCatalogueSync suppresses the single work fallback sync that runs + // when the direct insert produced no row. With it set, the core looks + // for the row once instead of polling, since nothing it started could + // still create it. False keeps the handler's behaviour. + SkipCatalogueSync bool } // addBookResult is what an Add Book produced. @@ -432,7 +440,7 @@ func (h *AuthorHandler) addBookCore(ctx context.Context, req addBookParams) (add // sync ran, restricted to the work the user actually picked, so the poll // below can still succeed without the rest of the bibliography riding // along (#1816). - if existing, _ := h.books.GetByForeignID(ctx, req.ForeignBookID); existing == nil { + if existing, _ := h.books.GetByForeignID(ctx, req.ForeignBookID); existing == nil && !req.SkipCatalogueSync { // mediaType only fills a format the provider left blank, and step 3 // below applies the request's explicit choice to whatever row the poll // finds — so the default is the right value to pass here. It no longer @@ -448,6 +456,9 @@ func (h *AuthorHandler) addBookCore(ctx context.Context, req addBookParams) (add // 2. Poll until the book appears (the single-work fallback, if it ran, // creates it asynchronously). deadline := time.Now().Add(15 * time.Second) + if req.SkipCatalogueSync { + deadline = time.Now() + } var book *models.Book for { b, _ := h.books.GetByForeignID(ctx, req.ForeignBookID) @@ -486,6 +497,9 @@ func (h *AuthorHandler) addBookCore(ctx context.Context, req addBookParams) (add // Re-evaluate status on a change so e.g. adding an already-imported ebook // as 'both' flips it back to wanted for the missing format (#1148). book.Monitored = true + if req.Monitored != nil { + book.Monitored = *req.Monitored + } if req.MediaType != "" && book.MediaType != req.MediaType { book.MediaType = req.MediaType reevaluateBookStatus(book) diff --git a/internal/api/add_book_core_test.go b/internal/api/add_book_core_test.go new file mode 100644 index 000000000..64d25bbb6 --- /dev/null +++ b/internal/api/add_book_core_test.go @@ -0,0 +1,360 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/jobs" + "github.com/vavallee/bindery/internal/metadata" + "github.com/vavallee/bindery/internal/models" +) + +// worksCountingProvider counts the author works calls a catalogue sync makes, +// so a test can tell whether addBookCore or createAuthorCore started one. +type worksCountingProvider struct { + *stubMetaProvider + worksCalls atomic.Int32 +} + +func (p *worksCountingProvider) GetAuthorWorks(ctx context.Context, id string) ([]models.Book, error) { + p.worksCalls.Add(1) + return p.stubMetaProvider.GetAuthorWorks(ctx, id) +} + +func (p *worksCountingProvider) GetAuthorWorksByName(ctx context.Context, name string) ([]models.Book, error) { + p.worksCalls.Add(1) + return p.stubMetaProvider.GetAuthorWorksByName(ctx, name) +} + +type coreFixture struct { + h *AuthorHandler + authors *db.AuthorRepo + books *db.BookRepo + jobs *jobs.Group +} + +// newCoreFixture wires an AuthorHandler over a fresh in memory database with +// a jobs group, so a test can drain every catalogue sync the call started +// before it counts provider calls. +func newCoreFixture(t *testing.T, provider metadata.Provider) coreFixture { + t.Helper() + database, err := db.OpenMemory() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + database.SetMaxOpenConns(1) + + authorRepo := db.NewAuthorRepo(database) + bookRepo := db.NewBookRepo(database) + profileRepo := db.NewMetadataProfileRepo(database) + group := jobs.NewGroup(context.Background()) + t.Cleanup(func() { group.Shutdown(5 * time.Second) }) + h := NewAuthorHandler(authorRepo, nil, bookRepo, nil, + metadata.NewAggregator(provider), nil, profileRepo, nil).WithJobs(group) + return coreFixture{h: h, authors: authorRepo, books: bookRepo, jobs: group} +} + +// drain waits for every job the handler started and fails on a straggler. +func (f coreFixture) drain(t *testing.T) { + t.Helper() + if left := f.jobs.Shutdown(5 * time.Second); len(left) != 0 { + t.Fatalf("jobs still running after drain: %v", left) + } +} + +func seedWells(t *testing.T, authors *db.AuthorRepo) *models.Author { + t.Helper() + author := &models.Author{ + ForeignID: "OL39307A", Name: "H. G. Wells", SortName: "Wells, H. G.", + MetadataProvider: "openlibrary", Monitored: true, + MonitorMode: models.AuthorMonitorModeAll, + } + if err := authors.Create(context.Background(), author); err != nil { + t.Fatal(err) + } + return author +} + +var wellsAddBookParams = addBookParams{ + ForeignBookID: "OL27482W", + ForeignAuthorID: "OL39307A", + AuthorName: "H. G. Wells", +} + +// Default params must give what the handler gives for the same request: the +// same row, monitored, filed under a newly created author. +func TestAddBookCore_DefaultParamsMatchHandler(t *testing.T) { + viaHandler := newCoreFixture(t, addBookBackCatalogueStub(false)) + body, _ := json.Marshal(map[string]any{ + "foreignBookId": wellsAddBookParams.ForeignBookID, + "foreignAuthorId": wellsAddBookParams.ForeignAuthorID, + "authorName": wellsAddBookParams.AuthorName, + }) + rec := httptest.NewRecorder() + viaHandler.h.AddBook(rec, httptest.NewRequest(http.MethodPost, "/api/v1/author/book", bytes.NewReader(body))) + if rec.Code != http.StatusCreated { + t.Fatalf("handler: expected 201, got %d: %s", rec.Code, rec.Body.String()) + } + var want models.Book + if err := json.Unmarshal(rec.Body.Bytes(), &want); err != nil { + t.Fatal(err) + } + + viaCore := newCoreFixture(t, addBookBackCatalogueStub(false)) + res, err := viaCore.h.addBookCore(context.Background(), wellsAddBookParams) + if err != nil { + t.Fatalf("core: %v", err) + } + got := res.Book + if got == nil { + t.Fatal("core returned no book") + } + if got.ForeignID != want.ForeignID || got.Title != want.Title || got.Monitored != want.Monitored || + got.MediaType != want.MediaType || got.Status != want.Status || got.OwnerUserID != want.OwnerUserID { + t.Errorf("core book = %+v\nhandler book = %+v", *got, want) + } + if !got.Monitored { + t.Error("default params must leave the book monitored") + } + if !res.BookCreated || !res.AuthorCreated { + t.Errorf("BookCreated=%v AuthorCreated=%v, want both true for a new author and book", res.BookCreated, res.AuthorCreated) + } + if res.Author == nil || res.Author.ForeignID != "OL39307A" || got.AuthorID != res.Author.ID { + t.Errorf("result author = %+v, book author id %d", res.Author, got.AuthorID) + } + stored, err := viaCore.books.GetByForeignID(context.Background(), "OL27482W") + if err != nil || stored == nil || stored.ID != got.ID || !stored.Monitored { + t.Errorf("stored row = %+v err=%v, want the returned monitored row", stored, err) + } + + // A second add of the same book is refused with the existing row, which + // the handler turns into its 409. + _, err = viaCore.h.addBookCore(context.Background(), wellsAddBookParams) + var inLibrary *bookInLibraryError + if !errors.As(err, &inLibrary) || inLibrary.Book == nil || inLibrary.Book.ID != got.ID { + t.Fatalf("re-add error = %v, want bookInLibraryError for row %d", err, got.ID) + } + rec = httptest.NewRecorder() + viaCore.h.writeAddBookError(rec, httptest.NewRequest(http.MethodPost, "/api/v1/author/book", nil), err) + if rec.Code != http.StatusConflict { + t.Errorf("mapped re-add status = %d, want 409", rec.Code) + } +} + +func TestAddBookCore_ExistingAuthorIsNotReportedCreated(t *testing.T) { + f := newCoreFixture(t, addBookBackCatalogueStub(false)) + author := seedWells(t, f.authors) + + res, err := f.h.addBookCore(context.Background(), wellsAddBookParams) + if err != nil { + t.Fatal(err) + } + if res.AuthorCreated { + t.Error("AuthorCreated = true for an author already in the library") + } + if !res.BookCreated { + t.Error("BookCreated = false for a book the direct insert created") + } + if res.Author == nil || res.Author.ID != author.ID { + t.Errorf("result author = %+v, want id %d", res.Author, author.ID) + } +} + +func TestAddBookCore_MonitoredFalseCreatesUnmonitoredBook(t *testing.T) { + f := newCoreFixture(t, addBookBackCatalogueStub(false)) + p := wellsAddBookParams + unmonitored := false + p.Monitored = &unmonitored + + res, err := f.h.addBookCore(context.Background(), p) + if err != nil { + t.Fatal(err) + } + if res.Book.Monitored { + t.Error("returned book is monitored, want unmonitored") + } + stored, err := f.books.GetByForeignID(context.Background(), "OL27482W") + if err != nil || stored == nil { + t.Fatalf("book not stored: err=%v", err) + } + if stored.Monitored { + t.Error("stored book is monitored, want unmonitored") + } + if !res.BookCreated { + t.Error("BookCreated = false for a newly inserted book") + } +} + +// When the direct insert fails, the default path falls back to a single work +// catalogue sync. SkipCatalogueSync must start no sync and answer at once +// instead of polling the full 15 seconds. +func TestAddBookCore_SkipCatalogueSyncFiresNoFetch(t *testing.T) { + t.Run("default runs the fallback sync", func(t *testing.T) { + provider := &worksCountingProvider{stubMetaProvider: addBookBackCatalogueStub(true)} + f := newCoreFixture(t, provider) + seedWells(t, f.authors) + + res, err := f.h.addBookCore(context.Background(), wellsAddBookParams) + if err != nil { + t.Fatalf("expected the fallback to create the book, got %v", err) + } + f.drain(t) + if provider.worksCalls.Load() == 0 { + t.Error("fallback made no author works call; the control case proves nothing") + } + if !res.BookCreated { + t.Error("BookCreated = false for a book the fallback sync created") + } + }) + + t.Run("skip starts no sync", func(t *testing.T) { + provider := &worksCountingProvider{stubMetaProvider: addBookBackCatalogueStub(true)} + f := newCoreFixture(t, provider) + seedWells(t, f.authors) + p := wellsAddBookParams + p.SkipCatalogueSync = true + + start := time.Now() + _, err := f.h.addBookCore(context.Background(), p) + elapsed := time.Since(start) + if !errors.Is(err, errAddBookNotFound) { + t.Fatalf("err = %v, want errAddBookNotFound", err) + } + f.drain(t) + if n := provider.worksCalls.Load(); n != 0 { + t.Errorf("author works calls = %d, want 0", n) + } + if got, _ := f.books.GetByForeignID(context.Background(), "OL27482W"); got != nil { + t.Errorf("book created without a sync: %+v", got) + } + if elapsed > 5*time.Second { + t.Errorf("skip path took %v; it should not poll", elapsed) + } + }) +} + +func wellsCreateProvider() *worksCountingProvider { + stub := addBookBackCatalogueStub(false) + stub.author = &models.Author{ + ForeignID: "OL39307A", Name: "H. G. Wells", SortName: "Wells, H. G.", + MetadataProvider: "openlibrary", + } + return &worksCountingProvider{stubMetaProvider: stub} +} + +var wellsCreateParams = createAuthorParams{ + ForeignID: "OL39307A", + Name: "H. G. Wells", + Monitored: true, +} + +func TestCreateAuthorCore_DefaultParamsMatchHandler(t *testing.T) { + viaHandler := newCoreFixture(t, wellsCreateProvider()) + body, _ := json.Marshal(map[string]any{ + "foreignAuthorId": wellsCreateParams.ForeignID, + "authorName": wellsCreateParams.Name, + "monitored": wellsCreateParams.Monitored, + }) + rec := httptest.NewRecorder() + viaHandler.h.Create(rec, httptest.NewRequest(http.MethodPost, "/api/v1/author", bytes.NewReader(body))) + if rec.Code != http.StatusCreated { + t.Fatalf("handler: expected 201, got %d: %s", rec.Code, rec.Body.String()) + } + var want models.Author + if err := json.Unmarshal(rec.Body.Bytes(), &want); err != nil { + t.Fatal(err) + } + viaHandler.drain(t) + + provider := wellsCreateProvider() + viaCore := newCoreFixture(t, provider) + res, err := viaCore.h.createAuthorCore(context.Background(), wellsCreateParams) + if err != nil { + t.Fatalf("core: %v", err) + } + viaCore.drain(t) + got := res.Author + if !res.Created || got == nil { + t.Fatalf("result = %+v, want a created author", res) + } + if got.ForeignID != want.ForeignID || got.Name != want.Name || got.Monitored != want.Monitored || + got.MonitorMode != want.MonitorMode || got.MonitorNewItems != want.MonitorNewItems { + t.Errorf("core author = %+v\nhandler author = %+v", *got, want) + } + if provider.worksCalls.Load() == 0 { + t.Error("default params started no catalogue sync") + } + + // Adding the same author again is the handler's 409 with the canonical row. + _, err = viaCore.h.createAuthorCore(context.Background(), wellsCreateParams) + var conflict *authorConflictError + if !errors.As(err, &conflict) || conflict.Canonical == nil || conflict.Canonical.ID != got.ID { + t.Fatalf("re-create error = %v, want authorConflictError for author %d", err, got.ID) + } +} + +func TestCreateAuthorCore_SkipCatalogueSyncFiresNoFetch(t *testing.T) { + provider := wellsCreateProvider() + f := newCoreFixture(t, provider) + p := wellsCreateParams + p.SkipCatalogueSync = true + + res, err := f.h.createAuthorCore(context.Background(), p) + if err != nil { + t.Fatal(err) + } + f.drain(t) + if !res.Created { + t.Error("Created = false, want a new author") + } + if n := provider.worksCalls.Load(); n != 0 { + t.Errorf("author works calls = %d, want 0", n) + } + books, err := f.books.ListByAuthor(context.Background(), res.Author.ID) + if err != nil { + t.Fatal(err) + } + if len(books) != 0 { + t.Errorf("skip created %d books, want 0", len(books)) + } +} + +// The handler maps every sentinel to the status and body it answered with +// before the split. This pins the table against the literal bodies. +func TestWriteAddBookError_Statuses(t *testing.T) { + h := &AuthorHandler{} + cases := []struct { + err error + status int + body string + }{ + {errAddBookForeignBookIDRequired, http.StatusBadRequest, "foreignBookId required"}, + {errAddBookInvalidMediaType, http.StatusBadRequest, "mediaType must be 'ebook', 'audiobook', or 'both'"}, + {errAddBookAuthorExists, http.StatusConflict, "author already exists"}, + {errAddBookAuthorUnresolved, http.StatusInternalServerError, "could not resolve author"}, + {errAddBookCancelled, http.StatusGatewayTimeout, "request cancelled"}, + {errAddBookHeldByAnotherUser, http.StatusConflict, "book is held by another user"}, + {&addBookLookupError{Err: errors.New("look up book metadata: boom")}, http.StatusBadGateway, "look up book metadata: boom"}, + {errors.New("disk on fire"), http.StatusInternalServerError, "internal server error"}, + } + for _, tc := range cases { + rec := httptest.NewRecorder() + h.writeAddBookError(rec, httptest.NewRequest(http.MethodPost, "/api/v1/author/book", nil), tc.err) + var got map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("%v: body %q: %v", tc.err, rec.Body.String(), err) + } + if rec.Code != tc.status || got["error"] != tc.body { + t.Errorf("%v: got %d %v, want %d %q", tc.err, rec.Code, got["error"], tc.status, tc.body) + } + } +} diff --git a/internal/api/create_author_core.go b/internal/api/create_author_core.go index e59bb2608..e5dd82468 100644 --- a/internal/api/create_author_core.go +++ b/internal/api/create_author_core.go @@ -34,6 +34,9 @@ type createAuthorParams struct { MonitorNewItems *string SearchOnAdd bool MediaType string + // SkipCatalogueSync creates or relinks the author without starting its + // catalogue sync. False keeps the handler's behaviour. + SkipCatalogueSync bool } // createAuthorResult is what an Add Author produced. @@ -177,7 +180,9 @@ func (h *AuthorHandler) createAuthorCore(ctx context.Context, req createAuthorPa // the snapshot read against this write (see fetchAuthorBooksAsync). cleanAuthorDescription(canonical) h.stampProviderMismatch(canonical) - h.fetchAuthorBooksAsync(canonical, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) + if !req.SkipCatalogueSync { + h.fetchAuthorBooksAsync(canonical, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) + } return createAuthorResult{Author: canonical}, nil } return createAuthorResult{}, &authorConflictError{Canonical: canonical, Message: "author name already resolves to an existing author — confirm merge"} @@ -222,7 +227,9 @@ func (h *AuthorHandler) createAuthorCore(ctx context.Context, req createAuthorPa // Fetch and store books for this author. Always populate the catalogue; // pass searchOnAdd so FetchAuthorBooks knows whether to also queue grabs. - h.fetchAuthorBooksAsync(author, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) + if !req.SkipCatalogueSync { + h.fetchAuthorBooksAsync(author, catalogueSyncOptions{autoSearch: req.SearchOnAdd, mediaType: mediaType}) + } telemetry.MarkFirst(ctx, h.settings, telemetry.SettingFirstAuthorAt) return createAuthorResult{Author: author, Created: true}, nil From 9825f2e6ac2efb5e54d005c379b0c8712ebcbcfd Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 01:43:58 -0300 Subject: [PATCH 03/18] feat(db): store unmatched library units with a lasting decision state Migration 088 adds unmatched_units, one row per book a library scan could not match, keyed by path so an ignore or an adoption survives the next scan. UnmatchedUnitRepo.ReconcileScan upserts in chunks of 500 and never moves an ignored row; it removes unseen pending rows and purges decisions unseen for 30 days, and does neither when the scan found no files. Adopt, undo and ignore claim a row with a single conditional UPDATE so two requests cannot both act on it. The list maps sort keys through a switch, binds every filter, and hydrates the books a page points at in one query. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- .../db/migrations/088_unmatched_units.sql | 77 +++ internal/db/unmatched_units.go | 514 ++++++++++++++++++ internal/db/unmatched_units_list.go | 269 +++++++++ internal/db/unmatched_units_test.go | 468 ++++++++++++++++ 4 files changed, 1328 insertions(+) create mode 100644 internal/db/migrations/088_unmatched_units.sql create mode 100644 internal/db/unmatched_units.go create mode 100644 internal/db/unmatched_units_list.go create mode 100644 internal/db/unmatched_units_test.go diff --git a/internal/db/migrations/088_unmatched_units.sql b/internal/db/migrations/088_unmatched_units.sql new file mode 100644 index 000000000..aa6b9c40d --- /dev/null +++ b/internal/db/migrations/088_unmatched_units.sql @@ -0,0 +1,77 @@ +-- +migrate Up + +-- Library adoption: the files a library scan could not match, one row per +-- book rather than per file. +-- +-- Until now the scan wrote its unmatched files into the library.lastScan +-- settings blob, capped at 1000 entries, one entry per file. A row there had +-- no identity, so nothing could be decided about it: no ignore state, no +-- grouping of a 193 track audiobook into one book (#2547), and no way to mark +-- a file as dealt with. This table gives each unit a stable identity keyed by +-- its path, so a decision (adopted, ignored) survives the next scan. +-- +-- state: +-- pending the scan still finds this unit and nobody has decided yet +-- adopting an adopt request holds the row (compare and swap claim) +-- adopted files registered against book_id by an admin +-- undoing an undo request holds the row +-- ignored an admin said this unit is not a book to track +-- +-- created_book_id and created_author_id are set only when the adopt request +-- itself inserted that row, which is what lets Undo remove them again without +-- touching a book or author that existed before. registered_paths_json lists +-- exactly the book_files paths the adopt inserted, for the same reason. +-- +-- member_paths_json holds the files that make up the unit. Adopt re-checks +-- each one on disk before registering anything, because the row may be hours +-- old by then. +-- +-- Times are written by the repository in one fixed width UTC layout so the +-- purge and in flight recovery comparisons below can compare them as text. + +CREATE TABLE unmatched_units ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + unit_path TEXT NOT NULL UNIQUE, + unit_kind TEXT NOT NULL CHECK(unit_kind IN ('file', 'folder')), + format TEXT NOT NULL CHECK(format IN ('ebook', 'audiobook')), + file_count INTEGER NOT NULL DEFAULT 1, + size_bytes INTEGER NOT NULL DEFAULT 0, + root_path TEXT NOT NULL DEFAULT '', + rel_path TEXT NOT NULL DEFAULT '', + author_folder TEXT NOT NULL DEFAULT '', + parsed_title TEXT NOT NULL DEFAULT '', + parsed_author TEXT NOT NULL DEFAULT '', + reason TEXT NOT NULL DEFAULT '', + search_key TEXT NOT NULL DEFAULT '', + member_paths_json TEXT NOT NULL DEFAULT '[]', + candidates_json TEXT NOT NULL DEFAULT '[]', + top_score REAL NOT NULL DEFAULT 0, + state TEXT NOT NULL DEFAULT 'pending' + CHECK(state IN ('pending', 'adopting', 'adopted', 'undoing', 'ignored')), + book_id INTEGER REFERENCES books(id) ON DELETE SET NULL, + created_book_id INTEGER REFERENCES books(id) ON DELETE SET NULL, + created_author_id INTEGER REFERENCES authors(id) ON DELETE SET NULL, + registered_paths_json TEXT NOT NULL DEFAULT '[]', + scan_generation INTEGER NOT NULL DEFAULT 0, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + resolved_at TEXT, + updated_at TEXT NOT NULL +); + +-- The default list: pending units, best suggestion first. +CREATE INDEX idx_unmatched_units_state_score ON unmatched_units(state, top_score DESC, rel_path); +-- The folder rail and the folder filter. +CREATE INDEX idx_unmatched_units_state_folder ON unmatched_units(state, author_folder, rel_path); +-- The reason facet and filter. +CREATE INDEX idx_unmatched_units_state_reason ON unmatched_units(state, reason); +-- Undo's "does any other row still point at this book" check. +CREATE INDEX idx_unmatched_units_book ON unmatched_units(book_id); + +-- +migrate Down + +DROP INDEX IF EXISTS idx_unmatched_units_book; +DROP INDEX IF EXISTS idx_unmatched_units_state_reason; +DROP INDEX IF EXISTS idx_unmatched_units_state_folder; +DROP INDEX IF EXISTS idx_unmatched_units_state_score; +DROP TABLE IF EXISTS unmatched_units; diff --git a/internal/db/unmatched_units.go b/internal/db/unmatched_units.go new file mode 100644 index 000000000..a735232c1 --- /dev/null +++ b/internal/db/unmatched_units.go @@ -0,0 +1,514 @@ +package db + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/vavallee/bindery/internal/textutil" +) + +// Unmatched unit states. See migration 088 for what each one means. +const ( + UnmatchedStatePending = "pending" + UnmatchedStateAdopting = "adopting" + UnmatchedStateAdopted = "adopted" + UnmatchedStateUndoing = "undoing" + UnmatchedStateIgnored = "ignored" +) + +// Unit kinds: a single file (an ebook, or loose audio at a library root), or +// a folder that stands for one book (an audiobook folder or a disc set). +const ( + UnmatchedKindFile = "file" + UnmatchedKindFolder = "folder" +) + +const ( + // reconcileChunkSize bounds how long one scan tail holds SQLite's single + // writer lock (P3). Imports and the queue get the lock between chunks. + reconcileChunkSize = 500 + // unmatchedPurgeAge is how long an ignored or adopted row may go unseen by + // a scan before it is purged. + unmatchedPurgeAge = 30 * 24 * time.Hour + // unmatchedInFlightTimeout recovers a row a crashed adopt or undo left + // claimed. No real request holds a row anywhere near this long. + unmatchedInFlightTimeout = 15 * time.Minute +) + +// unitTimeLayout is fixed width, so two stored times compare correctly as +// text. RFC3339Nano trims trailing zeros, and "05Z" sorts after "05.1Z". +const unitTimeLayout = "2006-01-02T15:04:05.000000000Z" + +func unitTime(t time.Time) string { return t.UTC().Format(unitTimeLayout) } + +// UnmatchedCandidate is one catalogue book the scan thought this unit might +// be, with the title similarity that put it there. +type UnmatchedCandidate struct { + BookID int64 `json:"bookId"` + Score float64 `json:"score"` +} + +// UnmatchedUnitScan is what a library scan reports for one unit. +type UnmatchedUnitScan struct { + UnitPath string + UnitKind string + Format string + FileCount int + SizeBytes int64 + RootPath string + RelPath string + AuthorFolder string + ParsedTitle string + ParsedAuthor string + Reason string + MemberPaths []string + Candidates []UnmatchedCandidate +} + +// UnmatchedUnit is one stored row. +type UnmatchedUnit struct { + ID int64 + UnitPath string + UnitKind string + Format string + FileCount int + SizeBytes int64 + RootPath string + RelPath string + AuthorFolder string + ParsedTitle string + ParsedAuthor string + Reason string + MemberPaths []string + Candidates []UnmatchedCandidate + TopScore float64 + State string + BookID int64 + CreatedBookID int64 + CreatedAuthorID int64 + RegisteredPaths []string + ScanGeneration int64 + FirstSeenAt time.Time + LastSeenAt time.Time + ResolvedAt *time.Time +} + +// ReconcileScanOptions tunes one ReconcileScan call. +type ReconcileScanOptions struct { + // StartedAt is when the scan read the tracked paths. An adopted row the + // scan still found unmatched returns to pending only when it was adopted + // before this, so an adopt that lands while a long walk is running is not + // undone by that walk's stale view. + StartedAt time.Time + // SkipDeletion keeps every existing row. Set when the scan found no files + // or failed, so an unmounted volume cannot erase anyone's decisions. + SkipDeletion bool + // SkipPurge keeps ignored and adopted rows past their age. Set when the + // scan was truncated, because a unit beyond the cap was not unseen. + SkipPurge bool + // Now is the clock; zero means time.Now. + Now time.Time +} + +// ReconcileScanResult reports what a reconcile did and the counts after it. +type ReconcileScanResult struct { + Generation int64 + Upserted int + RemovedPending int64 + Purged int64 + Pending int + Ignored int +} + +// UnmatchedUnitRepo stores library adoption rows. +type UnmatchedUnitRepo struct { + db *sql.DB +} + +// NewUnmatchedUnitRepo returns a repo over database. +func NewUnmatchedUnitRepo(database *sql.DB) *UnmatchedUnitRepo { + return &UnmatchedUnitRepo{db: database} +} + +const upsertUnmatchedUnitSQL = ` +INSERT INTO unmatched_units ( + unit_path, unit_kind, format, file_count, size_bytes, root_path, rel_path, + author_folder, parsed_title, parsed_author, reason, search_key, + member_paths_json, candidates_json, top_score, scan_generation, + first_seen_at, last_seen_at, updated_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(unit_path) DO UPDATE SET + unit_kind = excluded.unit_kind, + format = excluded.format, + file_count = excluded.file_count, + size_bytes = excluded.size_bytes, + root_path = excluded.root_path, + rel_path = excluded.rel_path, + author_folder = excluded.author_folder, + parsed_title = excluded.parsed_title, + parsed_author = excluded.parsed_author, + reason = excluded.reason, + search_key = excluded.search_key, + member_paths_json = excluded.member_paths_json, + candidates_json = excluded.candidates_json, + top_score = excluded.top_score, + scan_generation = excluded.scan_generation, + last_seen_at = excluded.last_seen_at, + updated_at = excluded.updated_at, + -- An adoption whose files the scan no longer sees as tracked has been + -- undone some other way (the book was deleted, a file row was removed), + -- so the unit is a decision to make again. Ignored stays ignored, and a + -- row adopted after this scan started is left alone. + state = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN 'pending' ELSE unmatched_units.state END, + book_id = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN NULL ELSE unmatched_units.book_id END, + created_book_id = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN NULL ELSE unmatched_units.created_book_id END, + created_author_id = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN NULL ELSE unmatched_units.created_author_id END, + registered_paths_json = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN '[]' ELSE unmatched_units.registered_paths_json END, + resolved_at = CASE WHEN unmatched_units.state = 'adopted' AND unmatched_units.resolved_at < ? + THEN NULL ELSE unmatched_units.resolved_at END` + +// ReconcileScan records one scan's units. Upserts are keyed by unit_path and +// committed every reconcileChunkSize rows (P3). A decision already taken on a +// row is kept: the upsert never moves an ignored row, and moves an adopted row +// back to pending only as described in upsertUnmatchedUnitSQL. Rows a crashed +// request left claimed are released first. After the upserts, one short +// transaction removes pending rows this scan did not see and purges ignored +// and adopted rows unseen for 30 days. The generation number keeps this correct if +// the process dies between chunks: the next scan's generation supersedes it. +func (r *UnmatchedUnitRepo) ReconcileScan(ctx context.Context, units []UnmatchedUnitScan, opts ReconcileScanOptions) (ReconcileScanResult, error) { + now := opts.Now + if now.IsZero() { + now = time.Now() + } + started := opts.StartedAt + if started.IsZero() { + started = now + } + var res ReconcileScanResult + if err := r.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(scan_generation), 0) + 1 FROM unmatched_units`).Scan(&res.Generation); err != nil { + return res, fmt.Errorf("unmatched units: next generation: %w", err) + } + nowText := unitTime(now) + startedText := unitTime(started) + + // Release rows a crashed adopt or undo left claimed. First, because the + // upserts below refresh updated_at, which is what dates a claim. + if _, err := r.db.ExecContext(ctx, + `UPDATE unmatched_units SET state = CASE state WHEN 'adopting' THEN 'pending' ELSE 'adopted' END, updated_at = ? + WHERE state IN ('adopting', 'undoing') AND updated_at < ?`, + nowText, unitTime(now.Add(-unmatchedInFlightTimeout))); err != nil { + return res, fmt.Errorf("unmatched units: release stale claims: %w", err) + } + + for start := 0; start < len(units); start += reconcileChunkSize { + end := min(start+reconcileChunkSize, len(units)) + if err := r.upsertChunk(ctx, units[start:end], res.Generation, nowText, startedText); err != nil { + return res, err + } + res.Upserted += end - start + } + + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return res, fmt.Errorf("unmatched units: begin cleanup: %w", err) + } + defer func() { _ = tx.Rollback() }() + if !opts.SkipDeletion { + out, err := tx.ExecContext(ctx, + `DELETE FROM unmatched_units WHERE state = 'pending' AND scan_generation < ?`, res.Generation) + if err != nil { + return res, fmt.Errorf("unmatched units: remove unseen pending: %w", err) + } + res.RemovedPending, _ = out.RowsAffected() + if !opts.SkipPurge { + out, err := tx.ExecContext(ctx, + `DELETE FROM unmatched_units WHERE state IN ('ignored', 'adopted') AND last_seen_at < ?`, + unitTime(now.Add(-unmatchedPurgeAge))) + if err != nil { + return res, fmt.Errorf("unmatched units: purge old decisions: %w", err) + } + res.Purged, _ = out.RowsAffected() + } + } + if err := tx.Commit(); err != nil { + return res, fmt.Errorf("unmatched units: commit cleanup: %w", err) + } + + summary, err := r.Summary(ctx) + if err != nil { + return res, err + } + res.Pending = summary.Pending + res.Ignored = summary.Ignored + return res, nil +} + +func (r *UnmatchedUnitRepo) upsertChunk(ctx context.Context, chunk []UnmatchedUnitScan, generation int64, nowText, startedText string) error { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("unmatched units: begin chunk: %w", err) + } + defer func() { _ = tx.Rollback() }() + stmt, err := tx.PrepareContext(ctx, upsertUnmatchedUnitSQL) + if err != nil { + return fmt.Errorf("unmatched units: prepare upsert: %w", err) + } + defer func() { _ = stmt.Close() }() + for i := range chunk { + u := &chunk[i] + members, err := json.Marshal(nonNilStrings(u.MemberPaths)) + if err != nil { + return fmt.Errorf("unmatched units: encode members: %w", err) + } + cands := u.Candidates + if cands == nil { + cands = []UnmatchedCandidate{} + } + candJSON, err := json.Marshal(cands) + if err != nil { + return fmt.Errorf("unmatched units: encode candidates: %w", err) + } + var top float64 + for _, c := range cands { + top = max(top, c.Score) + } + fileCount := max(u.FileCount, 1) + searchKey := textutil.FoldForSearch(u.ParsedTitle + " " + u.ParsedAuthor + " " + u.RelPath) + if _, err := stmt.ExecContext(ctx, + u.UnitPath, u.UnitKind, u.Format, fileCount, u.SizeBytes, u.RootPath, u.RelPath, + u.AuthorFolder, u.ParsedTitle, u.ParsedAuthor, u.Reason, searchKey, + string(members), string(candJSON), top, generation, + nowText, nowText, nowText, + startedText, startedText, startedText, startedText, startedText, startedText, + ); err != nil { + return fmt.Errorf("unmatched units: upsert %q: %w", u.UnitPath, err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("unmatched units: commit chunk: %w", err) + } + return nil +} + +func nonNilStrings(s []string) []string { + if s == nil { + return []string{} + } + return s +} + +// UnmatchedSummary is the headline counts, cheap enough for a nav badge. +type UnmatchedSummary struct { + Pending int `json:"pending"` + PendingFiles int `json:"pendingFiles"` + Ignored int `json:"ignored"` + Adopted int `json:"adopted"` +} + +// Summary counts rows by state. The adopting and undoing states count with +// the state they came from, since that is what the row still is to a viewer. +func (r *UnmatchedUnitRepo) Summary(ctx context.Context) (UnmatchedSummary, error) { + var s UnmatchedSummary + rows, err := r.db.QueryContext(ctx, + `SELECT state, COUNT(*), COALESCE(SUM(file_count), 0) FROM unmatched_units GROUP BY state`) + if err != nil { + return s, fmt.Errorf("unmatched units: summary: %w", err) + } + defer rows.Close() + for rows.Next() { + var state string + var n, files int + if err := rows.Scan(&state, &n, &files); err != nil { + return s, fmt.Errorf("unmatched units: summary scan: %w", err) + } + switch state { + case UnmatchedStatePending, UnmatchedStateAdopting: + s.Pending += n + s.PendingFiles += files + case UnmatchedStateAdopted, UnmatchedStateUndoing: + s.Adopted += n + case UnmatchedStateIgnored: + s.Ignored += n + } + } + return s, rows.Err() +} + +const unmatchedUnitColumns = `id, unit_path, unit_kind, format, file_count, size_bytes, root_path, rel_path, + author_folder, parsed_title, parsed_author, reason, member_paths_json, candidates_json, top_score, + state, book_id, created_book_id, created_author_id, registered_paths_json, scan_generation, + first_seen_at, last_seen_at, resolved_at` + +func scanUnmatchedUnit(scan func(...any) error) (*UnmatchedUnit, error) { + var u UnmatchedUnit + var members, cands, registered string + var bookID, createdBook, createdAuthor sql.NullInt64 + var first, last, resolved sql.NullString + if err := scan(&u.ID, &u.UnitPath, &u.UnitKind, &u.Format, &u.FileCount, &u.SizeBytes, &u.RootPath, &u.RelPath, + &u.AuthorFolder, &u.ParsedTitle, &u.ParsedAuthor, &u.Reason, &members, &cands, &u.TopScore, + &u.State, &bookID, &createdBook, &createdAuthor, ®istered, &u.ScanGeneration, + &first, &last, &resolved); err != nil { + return nil, err + } + u.BookID, u.CreatedBookID, u.CreatedAuthorID = bookID.Int64, createdBook.Int64, createdAuthor.Int64 + if err := json.Unmarshal([]byte(members), &u.MemberPaths); err != nil { + return nil, fmt.Errorf("unmatched unit %d members: %w", u.ID, err) + } + if err := json.Unmarshal([]byte(cands), &u.Candidates); err != nil { + return nil, fmt.Errorf("unmatched unit %d candidates: %w", u.ID, err) + } + if err := json.Unmarshal([]byte(registered), &u.RegisteredPaths); err != nil { + return nil, fmt.Errorf("unmatched unit %d registered paths: %w", u.ID, err) + } + u.FirstSeenAt = parseFlexibleTimeValue(first, "unmatched_units.first_seen_at") + u.LastSeenAt = parseFlexibleTimeValue(last, "unmatched_units.last_seen_at") + if t, err := parseFlexibleTime(resolved); err == nil { + u.ResolvedAt = t + } + return &u, nil +} + +// Get returns one row, or nil when there is none. +func (r *UnmatchedUnitRepo) Get(ctx context.Context, id int64) (*UnmatchedUnit, error) { + row := r.db.QueryRowContext(ctx, `SELECT `+unmatchedUnitColumns+` FROM unmatched_units WHERE id = ?`, id) + u, err := scanUnmatchedUnit(row.Scan) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("unmatched units: get %d: %w", id, err) + } + return u, nil +} + +// ClaimState moves a row from one state to another only if it is still in +// the first. It is the compare and swap every adopt, undo and ignore goes +// through (S9): a single UPDATE is atomic in SQLite, so of two requests racing +// for one row exactly one sees a row affected. +func (r *UnmatchedUnitRepo) ClaimState(ctx context.Context, id int64, from, to string) (bool, error) { + out, err := r.db.ExecContext(ctx, + `UPDATE unmatched_units SET state = ?, updated_at = ? WHERE id = ? AND state = ?`, + to, unitTime(time.Now()), id, from) + if err != nil { + return false, fmt.Errorf("unmatched units: claim %d %s to %s: %w", id, from, to, err) + } + n, err := out.RowsAffected() + return n == 1, err +} + +// AdoptionRecord is what a finished adopt wrote, and what undo reverses. +type AdoptionRecord struct { + BookID int64 + CreatedBookID int64 + CreatedAuthorID int64 + RegisteredPaths []string +} + +// CompleteAdoption moves a claimed row to adopted and records what the adopt +// did. It reports false when the row is no longer held by the adopt. +func (r *UnmatchedUnitRepo) CompleteAdoption(ctx context.Context, id int64, rec AdoptionRecord) (bool, error) { + paths, err := json.Marshal(nonNilStrings(rec.RegisteredPaths)) + if err != nil { + return false, err + } + now := unitTime(time.Now()) + out, err := r.db.ExecContext(ctx, ` + UPDATE unmatched_units + SET state = 'adopted', book_id = ?, created_book_id = ?, created_author_id = ?, + registered_paths_json = ?, resolved_at = ?, updated_at = ? + WHERE id = ? AND state = 'adopting'`, + nullID(rec.BookID), nullID(rec.CreatedBookID), nullID(rec.CreatedAuthorID), string(paths), now, now, id) + if err != nil { + return false, fmt.Errorf("unmatched units: complete adoption %d: %w", id, err) + } + n, err := out.RowsAffected() + return n == 1, err +} + +// CompleteUndo returns a claimed row to pending and clears the adoption. +func (r *UnmatchedUnitRepo) CompleteUndo(ctx context.Context, id int64) (bool, error) { + out, err := r.db.ExecContext(ctx, ` + UPDATE unmatched_units + SET state = 'pending', book_id = NULL, created_book_id = NULL, created_author_id = NULL, + registered_paths_json = '[]', resolved_at = NULL, updated_at = ? + WHERE id = ? AND state = 'undoing'`, unitTime(time.Now()), id) + if err != nil { + return false, fmt.Errorf("unmatched units: complete undo %d: %w", id, err) + } + n, err := out.RowsAffected() + return n == 1, err +} + +func nullID(id int64) any { + if id <= 0 { + return nil + } + return id +} + +// IgnorePending marks pending rows ignored, selected by id or by author +// folder (exactly one of the two). Rows in any other state are left alone. +func (r *UnmatchedUnitRepo) IgnorePending(ctx context.Context, ids []int64, authorFolder string) (int64, error) { + now := unitTime(time.Now()) + var out sql.Result + var err error + switch { + case len(ids) > 0: + ph := make([]string, len(ids)) + args := make([]any, 0, len(ids)+1) + args = append(args, now) + for i, id := range ids { + ph[i] = "?" + args = append(args, id) + } + //nolint:gosec // G202: the IN list is generated ? placeholders; every id is bound + out, err = r.db.ExecContext(ctx, + `UPDATE unmatched_units SET state = 'ignored', updated_at = ? WHERE state = 'pending' AND id IN (`+strings.Join(ph, ",")+`)`, + args...) + case authorFolder != "": + out, err = r.db.ExecContext(ctx, + `UPDATE unmatched_units SET state = 'ignored', updated_at = ? WHERE state = 'pending' AND author_folder = ?`, + now, authorFolder) + default: + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("unmatched units: ignore: %w", err) + } + return out.RowsAffected() +} + +// BookReferencedElsewhere reports whether any row other than exceptID still +// points at bookID, as its adopted book or as a book it created. +func (r *UnmatchedUnitRepo) BookReferencedElsewhere(ctx context.Context, bookID, exceptID int64) (bool, error) { + var one int + err := r.db.QueryRowContext(ctx, + `SELECT 1 FROM unmatched_units WHERE id <> ? AND (book_id = ? OR created_book_id = ?) LIMIT 1`, + exceptID, bookID, bookID).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +// AuthorReferencedElsewhere is BookReferencedElsewhere for a created author. +func (r *UnmatchedUnitRepo) AuthorReferencedElsewhere(ctx context.Context, authorID, exceptID int64) (bool, error) { + var one int + err := r.db.QueryRowContext(ctx, + `SELECT 1 FROM unmatched_units WHERE id <> ? AND created_author_id = ? LIMIT 1`, + exceptID, authorID).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} diff --git a/internal/db/unmatched_units_list.go b/internal/db/unmatched_units_list.go new file mode 100644 index 000000000..5c6c70fdd --- /dev/null +++ b/internal/db/unmatched_units_list.go @@ -0,0 +1,269 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/vavallee/bindery/internal/textutil" +) + +// UnmatchedListQuery filters and orders a page of unmatched units. Every +// field is bound or mapped through a switch; none reaches SQL text (S9). +type UnmatchedListQuery struct { + // State is pending (default), ignored or adopted. + State string + Reason string + AuthorFolder string + Format string + Search string + // Sort is score (default), title, folder, files, size or seen. + Sort string + // Dir is asc or desc; empty picks the sort's natural direction. + Dir string + Limit int + Offset int +} + +// unmatchedListState maps a requested state to the stored state it lists. +// The in flight states (adopting, undoing) last for one request and are not +// listed, which keeps every list on a single state equality and so in index +// order (P4). A row a crashed request left claimed is released by the next +// scan's reconcile. +func unmatchedListState(state string) string { + switch state { + case UnmatchedStateIgnored, UnmatchedStateAdopted: + return state + default: + return UnmatchedStatePending + } +} + +// unmatchedOrderBy maps a sort key and direction to a fixed ORDER BY clause. +func unmatchedOrderBy(sort, dir string) string { + desc := func(natural bool) string { + switch dir { + case "asc": + return "ASC" + case "desc": + return "DESC" + } + if natural { + return "DESC" + } + return "ASC" + } + switch sort { + case "title": + return "parsed_title " + desc(false) + ", id" + case "folder": + return "author_folder " + desc(false) + ", rel_path " + desc(false) + case "files": + return "file_count " + desc(true) + ", rel_path" + case "size": + return "size_bytes " + desc(true) + ", rel_path" + case "seen": + return "first_seen_at " + desc(true) + ", id" + default: + return "top_score " + desc(true) + ", rel_path" + } +} + +func unmatchedWhere(q UnmatchedListQuery, includeFacetFilters bool) (string, []any) { + where := "state = ?" + args := []any{unmatchedListState(q.State)} + if includeFacetFilters { + if q.Reason != "" { + where += " AND reason = ?" + args = append(args, q.Reason) + } + if q.AuthorFolder != "" { + where += " AND author_folder = ?" + args = append(args, q.AuthorFolder) + } + if q.Format != "" { + where += " AND format = ?" + args = append(args, q.Format) + } + } + // A substring match over at most 20,000 rows (the scan's unit cap), one + // LIKE per folded word. FoldForSearch already drops LIKE's wildcards; the + // escape stays so that never depends on the fold. Acceptable at that size, and + // every other predicate above is index backed (P4). + if folded := textutil.FoldForSearch(q.Search); folded != "" { + for _, tok := range searchTokens(folded) { + where += " AND search_key LIKE ? ESCAPE '\\'" + args = append(args, "%"+escapeLike(tok)+"%") + } + } + return where, args +} + +// List returns one page of units and the total matching the filters. +func (r *UnmatchedUnitRepo) List(ctx context.Context, q UnmatchedListQuery) ([]UnmatchedUnit, int, error) { + where, args := unmatchedWhere(q, true) + var total int + if err := r.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM unmatched_units WHERE `+where, args...).Scan(&total); err != nil { + return nil, 0, fmt.Errorf("unmatched units: count: %w", err) + } + limit := q.Limit + if limit <= 0 || limit > 250 { + limit = 50 + } + offset := max(q.Offset, 0) + //nolint:gosec // G202: where is fixed predicates with bound values (unmatchedWhere) and the ORDER BY comes from the unmatchedOrderBy switch + rows, err := r.db.QueryContext(ctx, + `SELECT `+unmatchedUnitColumns+` FROM unmatched_units WHERE `+where+ + ` ORDER BY `+unmatchedOrderBy(q.Sort, q.Dir)+` LIMIT ? OFFSET ?`, + append(args, limit, offset)...) + if err != nil { + return nil, 0, fmt.Errorf("unmatched units: list: %w", err) + } + defer rows.Close() + out := make([]UnmatchedUnit, 0, limit) + for rows.Next() { + u, err := scanUnmatchedUnit(rows.Scan) + if err != nil { + return nil, 0, fmt.Errorf("unmatched units: list scan: %w", err) + } + out = append(out, *u) + } + return out, total, rows.Err() +} + +// FacetCount is one value of a facet and how many units carry it. +type FacetCount struct { + Value string `json:"value"` + Count int `json:"count"` +} + +// FolderFacet is one author folder in the structure rail. +type FolderFacet struct { + Folder string `json:"folder"` + Units int `json:"units"` + Files int `json:"files"` + // NotInLibrary counts units whose author matched no library author. When + // it equals Units the whole folder is one decision: add the author. + NotInLibrary int `json:"notInLibrary"` + // Author is the parsed author most units in the folder share. + Author string `json:"author"` +} + +// UnmatchedFacets are the grouped counts behind the filters and the rail. +type UnmatchedFacets struct { + Reasons []FacetCount `json:"reasons"` + Formats []FacetCount `json:"formats"` + Folders []FolderFacet `json:"folders"` +} + +// maxFolderFacets bounds the rail. The rail is for the big decisions; a long +// tail of one book folders is what the table and its folder filter are for. +const maxFolderFacets = 12 + +// Facets runs the three grouped counts for a state and search. The web asks +// for them only when that pair changes, not on every page turn (P4). +func (r *UnmatchedUnitRepo) Facets(ctx context.Context, q UnmatchedListQuery) (UnmatchedFacets, error) { + where, args := unmatchedWhere(q, false) + f := UnmatchedFacets{Reasons: []FacetCount{}, Formats: []FacetCount{}, Folders: []FolderFacet{}} + grouped := func(col string) ([]FacetCount, error) { + //nolint:gosec // G202: col is one of two literal column names below; where binds every value + rows, err := r.db.QueryContext(ctx, + `SELECT `+col+`, COUNT(*) FROM unmatched_units WHERE `+where+` GROUP BY `+col+` ORDER BY COUNT(*) DESC, `+col, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []FacetCount{} + for rows.Next() { + var fc FacetCount + if err := rows.Scan(&fc.Value, &fc.Count); err != nil { + return nil, err + } + out = append(out, fc) + } + return out, rows.Err() + } + var err error + if f.Reasons, err = grouped("reason"); err != nil { + return f, fmt.Errorf("unmatched units: reason facet: %w", err) + } + if f.Formats, err = grouped("format"); err != nil { + return f, fmt.Errorf("unmatched units: format facet: %w", err) + } + // The most common parsed author per folder comes from a correlated + // subquery that only runs for the handful of folders the LIMIT keeps. Its + // unqualified columns resolve to the innermost table, a. + //nolint:gosec // G202: where is fixed predicates with bound values (unmatchedWhere) + rows, err := r.db.QueryContext(ctx, ` + SELECT g.author_folder, g.units, g.files, g.not_in_library, + COALESCE((SELECT parsed_author FROM unmatched_units a + WHERE a.author_folder = g.author_folder AND a.parsed_author <> '' AND `+where+` + GROUP BY parsed_author ORDER BY COUNT(*) DESC LIMIT 1), '') + FROM ( + SELECT author_folder, COUNT(*) AS units, SUM(file_count) AS files, + SUM(CASE WHEN reason = 'author_not_in_library' THEN 1 ELSE 0 END) AS not_in_library + FROM unmatched_units WHERE author_folder <> '' AND `+where+` + GROUP BY author_folder ORDER BY units DESC, author_folder LIMIT ? + ) g`, append(append(append([]any{}, args...), args...), maxFolderFacets)...) + if err != nil { + return f, fmt.Errorf("unmatched units: folder facet: %w", err) + } + defer rows.Close() + for rows.Next() { + var ff FolderFacet + if err := rows.Scan(&ff.Folder, &ff.Units, &ff.Files, &ff.NotInLibrary, &ff.Author); err != nil { + return f, fmt.Errorf("unmatched units: folder facet scan: %w", err) + } + f.Folders = append(f.Folders, ff) + } + return f, rows.Err() +} + +// UnmatchedBookRef is the slice of a book the adoption list shows beside a +// suggestion or an adopted row. +type UnmatchedBookRef struct { + ID int64 `json:"id"` + Title string `json:"title"` + AuthorID int64 `json:"authorId"` + AuthorName string `json:"authorName"` + ImageURL string `json:"imageUrl,omitempty"` + Status string `json:"status"` + MediaType string `json:"mediaType"` + Monitored bool `json:"monitored"` +} + +// BookRefs loads the books a page of units points at in one IN query (P4, +// T5), whatever the number of rows and candidates on the page. +func (r *UnmatchedUnitRepo) BookRefs(ctx context.Context, ids []int64) (map[int64]UnmatchedBookRef, error) { + out := make(map[int64]UnmatchedBookRef, len(ids)) + if len(ids) == 0 { + return out, nil + } + ph := make([]string, len(ids)) + args := make([]any, len(ids)) + for i, id := range ids { + ph[i] = "?" + args[i] = id + } + //nolint:gosec // G202: the IN list is generated ? placeholders; every id is bound + rows, err := r.db.QueryContext(ctx, ` + SELECT b.id, b.title, b.author_id, COALESCE(a.name, ''), COALESCE(b.image_url, ''), + b.status, b.media_type, b.monitored + FROM books b LEFT JOIN authors a ON a.id = b.author_id + WHERE b.id IN (`+strings.Join(ph, ",")+`)`, args...) + if err != nil { + return nil, fmt.Errorf("unmatched units: book refs: %w", err) + } + defer rows.Close() + for rows.Next() { + var b UnmatchedBookRef + var authorID sql.NullInt64 + if err := rows.Scan(&b.ID, &b.Title, &authorID, &b.AuthorName, &b.ImageURL, &b.Status, &b.MediaType, &b.Monitored); err != nil { + return nil, fmt.Errorf("unmatched units: book refs scan: %w", err) + } + b.AuthorID = authorID.Int64 + out[b.ID] = b + } + return out, rows.Err() +} diff --git a/internal/db/unmatched_units_test.go b/internal/db/unmatched_units_test.go new file mode 100644 index 000000000..4e24511c3 --- /dev/null +++ b/internal/db/unmatched_units_test.go @@ -0,0 +1,468 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/vavallee/bindery/internal/models" +) + +func openUnmatchedRepo(t testing.TB) (*sql.DB, *UnmatchedUnitRepo) { + t.Helper() + database, err := OpenMemory() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = database.Close() }) + return database, NewUnmatchedUnitRepo(database) +} + +func scanUnit(path string) UnmatchedUnitScan { + return UnmatchedUnitScan{ + UnitPath: path, UnitKind: UnmatchedKindFile, Format: "ebook", FileCount: 1, SizeBytes: 10, + RootPath: "/lib", RelPath: strings.TrimPrefix(path, "/lib/"), AuthorFolder: "A", + ParsedTitle: "T " + path, ParsedAuthor: "A", Reason: "no_title_match", + MemberPaths: []string{path}, + } +} + +func unitByPath(t *testing.T, database *sql.DB, repo *UnmatchedUnitRepo, path string) *UnmatchedUnit { + t.Helper() + var id int64 + err := database.QueryRow(`SELECT id FROM unmatched_units WHERE unit_path = ?`, path).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + t.Fatal(err) + } + u, err := repo.Get(context.Background(), id) + if err != nil { + t.Fatal(err) + } + return u +} + +// TestReconcileScan_KeepsDecisions covers the ReconcileScan contract: a rescan +// never moves an ignored row, a unit the scan stops seeing is removed while +// pending and kept once decided, and the counts come back. +func TestReconcileScan_KeepsDecisions(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + + first := []UnmatchedUnitScan{scanUnit("/lib/A/one.epub"), scanUnit("/lib/A/two.epub"), scanUnit("/lib/A/three.epub")} + res, err := repo.ReconcileScan(ctx, first, ReconcileScanOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Pending != 3 || res.Generation != 1 { + t.Fatalf("first reconcile = %+v, want 3 pending in generation 1", res) + } + + two := unitByPath(t, database, repo, "/lib/A/two.epub") + if ok, err := repo.ClaimState(ctx, two.ID, UnmatchedStatePending, UnmatchedStateIgnored); err != nil || !ok { + t.Fatalf("ignore two: ok=%v err=%v", ok, err) + } + + // The second scan still sees two (ignored) and one, and no longer sees three. + updated := scanUnit("/lib/A/one.epub") + updated.ParsedTitle = "Renamed" + res, err = repo.ReconcileScan(ctx, []UnmatchedUnitScan{updated, scanUnit("/lib/A/two.epub")}, ReconcileScanOptions{}) + if err != nil { + t.Fatal(err) + } + if res.RemovedPending != 1 || res.Pending != 1 || res.Ignored != 1 { + t.Fatalf("second reconcile = %+v, want 1 removed, 1 pending, 1 ignored", res) + } + if got := unitByPath(t, database, repo, "/lib/A/two.epub"); got == nil || got.State != UnmatchedStateIgnored { + t.Fatalf("ignored row after rescan = %+v, want still ignored", got) + } + if got := unitByPath(t, database, repo, "/lib/A/three.epub"); got != nil { + t.Fatalf("vanished pending row survived: %+v", got) + } + if got := unitByPath(t, database, repo, "/lib/A/one.epub"); got.ParsedTitle != "Renamed" || got.ID == two.ID { + t.Fatalf("pending row not refreshed in place: %+v", got) + } +} + +// TestReconcileScan_ZeroFileScanDeletesNothing is the unmounted volume case: a +// scan that found nothing must not erase a single row or decision. +func TestReconcileScan_ZeroFileScanDeletesNothing(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/one.epub"), scanUnit("/lib/A/two.epub")}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + two := unitByPath(t, database, repo, "/lib/A/two.epub") + if _, err := repo.ClaimState(ctx, two.ID, UnmatchedStatePending, UnmatchedStateIgnored); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-90 * 24 * time.Hour) + if _, err := database.Exec(`UPDATE unmatched_units SET last_seen_at = ?`, unitTime(old)); err != nil { + t.Fatal(err) + } + res, err := repo.ReconcileScan(ctx, nil, ReconcileScanOptions{SkipDeletion: true}) + if err != nil { + t.Fatal(err) + } + if res.RemovedPending != 0 || res.Purged != 0 || res.Pending != 1 || res.Ignored != 1 { + t.Fatalf("zero file reconcile = %+v, want nothing removed", res) + } +} + +// TestReconcileScan_PurgesOldDecisions: ignored and adopted rows unseen for 30 +// days go, a truncated scan purges nothing, and a recent one stays. +func TestReconcileScan_PurgesOldDecisions(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + units := []UnmatchedUnitScan{scanUnit("/lib/A/old.epub"), scanUnit("/lib/A/recent.epub")} + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`UPDATE unmatched_units SET state = 'ignored'`); err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`UPDATE unmatched_units SET last_seen_at = ? WHERE unit_path = '/lib/A/old.epub'`, + unitTime(time.Now().Add(-31*24*time.Hour))); err != nil { + t.Fatal(err) + } + + res, err := repo.ReconcileScan(ctx, nil, ReconcileScanOptions{SkipPurge: true}) + if err != nil { + t.Fatal(err) + } + if res.Purged != 0 || res.Ignored != 2 { + t.Fatalf("truncated reconcile = %+v, want no purge", res) + } + res, err = repo.ReconcileScan(ctx, nil, ReconcileScanOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Purged != 1 || res.Ignored != 1 { + t.Fatalf("reconcile = %+v, want the 31 day old row purged", res) + } + if unitByPath(t, database, repo, "/lib/A/recent.epub") == nil { + t.Fatal("recent ignored row purged") + } +} + +// TestReconcileScan_AdoptedRowSeenAgain: an adoption finished before the scan +// started whose unit is unmatched again was undone elsewhere, so it returns to +// pending; one finished while the scan ran is left alone. +func TestReconcileScan_AdoptedRowSeenAgain(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + units := []UnmatchedUnitScan{scanUnit("/lib/A/before.epub"), scanUnit("/lib/A/during.epub")} + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + for _, p := range []string{"/lib/A/before.epub", "/lib/A/during.epub"} { + u := unitByPath(t, database, repo, p) + if ok, _ := repo.ClaimState(ctx, u.ID, UnmatchedStatePending, UnmatchedStateAdopting); !ok { + t.Fatal("claim failed") + } + if ok, err := repo.CompleteAdoption(ctx, u.ID, AdoptionRecord{RegisteredPaths: []string{p}}); err != nil || !ok { + t.Fatalf("complete: ok=%v err=%v", ok, err) + } + } + scanStart := time.Now() + if _, err := database.Exec(`UPDATE unmatched_units SET resolved_at = ? WHERE unit_path = '/lib/A/before.epub'`, + unitTime(scanStart.Add(-time.Hour))); err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`UPDATE unmatched_units SET resolved_at = ? WHERE unit_path = '/lib/A/during.epub'`, + unitTime(scanStart.Add(time.Minute))); err != nil { + t.Fatal(err) + } + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{StartedAt: scanStart}); err != nil { + t.Fatal(err) + } + if got := unitByPath(t, database, repo, "/lib/A/before.epub"); got.State != UnmatchedStatePending || len(got.RegisteredPaths) != 0 { + t.Fatalf("stale adoption = %+v, want pending with nothing registered", got) + } + if got := unitByPath(t, database, repo, "/lib/A/during.epub"); got.State != UnmatchedStateAdopted { + t.Fatalf("adoption during the scan = %+v, want still adopted", got) + } +} + +// TestReconcileScan_ChunksAndStaleClaims: more than one chunk of units lands, +// and a row a crashed request left claimed is released. +func TestReconcileScan_ChunksAndStaleClaims(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + units := make([]UnmatchedUnitScan, reconcileChunkSize*2+7) + for i := range units { + units[i] = scanUnit(fmt.Sprintf("/lib/A/%04d.epub", i)) + } + res, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Upserted != len(units) || res.Pending != len(units) { + t.Fatalf("reconcile = %+v, want %d rows", res, len(units)) + } + if _, err := database.Exec(`UPDATE unmatched_units SET state = 'adopting', updated_at = ? WHERE unit_path = '/lib/A/0000.epub'`, + unitTime(time.Now().Add(-time.Hour))); err != nil { + t.Fatal(err) + } + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + if got := unitByPath(t, database, repo, "/lib/A/0000.epub"); got.State != UnmatchedStatePending { + t.Fatalf("stale claim = %q, want released to pending", got.State) + } +} + +// TestClaimState_ExactlyOneWinner is T3 at the repository: many goroutines +// racing to claim one row, one wins. +func TestClaimState_ExactlyOneWinner(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/x.epub")}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + u := unitByPath(t, database, repo, "/lib/A/x.epub") + var wg sync.WaitGroup + var mu sync.Mutex + wins := 0 + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + ok, err := repo.ClaimState(ctx, u.ID, UnmatchedStatePending, UnmatchedStateAdopting) + if err != nil { + t.Error(err) + } + if ok { + mu.Lock() + wins++ + mu.Unlock() + } + }() + } + wg.Wait() + if wins != 1 { + t.Fatalf("claims won = %d, want 1", wins) + } +} + +// TestUnmatchedList_FiltersSortsAndEscapes covers the list parameters S9 names: +// an unknown sort falls back, search is folded and bound, and the folder +// filter is bound. +func TestUnmatchedList_FiltersSortsAndEscapes(t *testing.T) { + ctx := context.Background() + _, repo := openUnmatchedRepo(t) + a := scanUnit("/lib/Weir/Hail Mary.epub") + a.ParsedTitle, a.AuthorFolder, a.Candidates = "Project Hail Mary", "Weir", []UnmatchedCandidate{{BookID: 1, Score: 0.7}} + b := scanUnit("/lib/Other/100% Done.epub") + b.ParsedTitle, b.AuthorFolder, b.FileCount = "100% Done", "Other", 3 + c := scanUnit("/lib/Other/100 Done.epub") + c.ParsedTitle, c.AuthorFolder, c.Reason = "100 Done", "Other", "author_not_in_library" + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{a, b, c}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + + items, total, err := repo.List(ctx, UnmatchedListQuery{Sort: "score); DROP TABLE unmatched_units; --"}) + if err != nil || total != 3 || items[0].ParsedTitle != "Project Hail Mary" { + t.Fatalf("default list = %d items, total %d, err %v", len(items), total, err) + } + items, _, err = repo.List(ctx, UnmatchedListQuery{Search: "hail MARY"}) + if err != nil || len(items) != 1 || items[0].ParsedTitle != "Project Hail Mary" { + t.Fatalf("search = %+v err %v", items, err) + } + // A bare wildcard folds away to no term at all: every row, not an error + // and not a pattern. + if _, total, err = repo.List(ctx, UnmatchedListQuery{Search: "%_"}); err != nil || total != 3 { + t.Fatalf("wildcard search total = %d err %v, want 3", total, err) + } + items, _, err = repo.List(ctx, UnmatchedListQuery{AuthorFolder: "Other' OR '1'='1", Sort: "files"}) + if err != nil || len(items) != 0 { + t.Fatalf("injected folder matched %d rows", len(items)) + } + items, _, err = repo.List(ctx, UnmatchedListQuery{AuthorFolder: "Other", Sort: "files"}) + if err != nil || len(items) != 2 || items[0].FileCount != 3 { + t.Fatalf("folder list sorted by files = %+v err %v", items, err) + } + + f, err := repo.Facets(ctx, UnmatchedListQuery{}) + if err != nil { + t.Fatal(err) + } + if len(f.Folders) != 2 || f.Folders[0].Folder != "Other" || f.Folders[0].Units != 2 || f.Folders[0].Files != 4 || f.Folders[0].NotInLibrary != 1 { + t.Fatalf("folder facets = %+v", f.Folders) + } + if len(f.Reasons) != 2 || len(f.Formats) != 1 { + t.Fatalf("facets = %+v", f) + } +} + +func queryPlan(t *testing.T, database *sql.DB, query string, args ...any) string { + t.Helper() + rows, err := database.Query(`EXPLAIN QUERY PLAN `+query, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return strings.Join(plan, " | ") +} + +// TestUnmatchedList_QueryPlansUseIndexes is P4: the default list and every sort +// search an index for the state filter instead of scanning the table. +func TestUnmatchedList_QueryPlansUseIndexes(t *testing.T) { + database, _ := openUnmatchedRepo(t) + for _, sort := range []string{"", "score", "title", "folder", "files", "size", "seen"} { + q := UnmatchedListQuery{Sort: sort} + where, args := unmatchedWhere(q, true) + joined := queryPlan(t, database, `SELECT id FROM unmatched_units WHERE `+where+ + ` ORDER BY `+unmatchedOrderBy(q.Sort, q.Dir)+` LIMIT 50`, args...) + if !strings.Contains(joined, "INDEX idx_unmatched_units_state") || strings.Contains(joined, "SCAN unmatched_units") { + t.Errorf("sort %q plan does not search a state index: %s", sort, joined) + } + if (sort == "" || sort == "score" || sort == "folder") && strings.Contains(joined, "TEMP B-TREE") { + t.Errorf("sort %q should be served in index order: %s", sort, joined) + } + } +} + +// TestBookRefs_OneQueryPerPage hydrates many ids through a single statement. +func TestBookRefs_OneQueryPerPage(t *testing.T) { + ctx := context.Background() + database, repo := openUnmatchedRepo(t) + authors := NewAuthorRepo(database) + books := NewBookRepo(database) + author := &models.Author{ForeignID: "ol:a", Name: "Ann Leckie", SortName: "Leckie, Ann", MetadataProvider: "openlibrary"} + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + var ids []int64 + for i := range 5 { + b := &models.Book{ForeignID: fmt.Sprintf("ol:b%d", i), AuthorID: author.ID, Title: fmt.Sprintf("Book %d", i), + Status: models.BookStatusWanted, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := books.Create(ctx, b); err != nil { + t.Fatal(err) + } + ids = append(ids, b.ID) + } + refs, err := repo.BookRefs(ctx, append(ids, 99999)) + if err != nil { + t.Fatal(err) + } + if len(refs) != 5 || refs[ids[2]].AuthorName != "Ann Leckie" || refs[ids[2]].Title != "Book 2" { + t.Fatalf("refs = %+v", refs) + } +} + +// TestMigrate088_OverPopulatedDatabase is T4: 088 applied to a database that +// already holds authors, books, notifications and users adds an empty table, +// leaves every existing row alone, and clears a book reference on delete +// instead of blocking it. +func TestMigrate088_OverPopulatedDatabase(t *testing.T) { + ctx := context.Background() + database, err := OpenMemory() + if err != nil { + t.Fatal(err) + } + defer database.Close() + + for _, stmt := range []string{ + `DROP INDEX idx_unmatched_units_book`, `DROP INDEX idx_unmatched_units_state_reason`, + `DROP INDEX idx_unmatched_units_state_folder`, `DROP INDEX idx_unmatched_units_state_score`, + `DROP TABLE unmatched_units`, + } { + if _, err := database.Exec(stmt); err != nil { + t.Fatalf("%s: %v", stmt, err) + } + } + + authors := NewAuthorRepo(database) + books := NewBookRepo(database) + author := &models.Author{ForeignID: "ol:a", Name: "Ann Leckie", SortName: "Leckie, Ann", MetadataProvider: "openlibrary"} + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + book := &models.Book{ForeignID: "ol:b", AuthorID: author.ID, Title: "Ancillary Justice", + Status: models.BookStatusWanted, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := books.Create(ctx, book); err != nil { + t.Fatal(err) + } + if err := NewNotificationRepo(database).Create(ctx, &models.Notification{Name: "hook", Type: "webhook", URL: "http://example.invalid", Enabled: true}); err != nil { + t.Fatal(err) + } + if _, err := NewUserRepo(database).Create(ctx, "reader", "hash"); err != nil { + t.Fatal(err) + } + + v := migrationVersionForTest(t, "088_unmatched_units.sql") + if _, err := database.Exec(`DELETE FROM schema_migrations WHERE version = ?`, v); err != nil { + t.Fatal(err) + } + if err := migrate(database); err != nil { + t.Fatalf("rerun migration 088: %v", err) + } + + var n int + if err := database.QueryRow(`SELECT COUNT(*) FROM unmatched_units`).Scan(&n); err != nil || n != 0 { + t.Fatalf("unmatched_units after migration: n=%d err=%v", n, err) + } + for table, want := range map[string]int{"authors": 1, "books": 1, "notifications": 1, "users": 1} { + if err := database.QueryRow(`SELECT COUNT(*) FROM ` + table).Scan(&n); err != nil || n != want { + t.Errorf("%s rows after migration = %d (err %v), want %d", table, n, err, want) + } + } + + repo := NewUnmatchedUnitRepo(database) + if _, err := repo.ReconcileScan(ctx, []UnmatchedUnitScan{scanUnit("/lib/A/x.epub")}, ReconcileScanOptions{}); err != nil { + t.Fatal(err) + } + u := unitByPath(t, database, repo, "/lib/A/x.epub") + if u.State != UnmatchedStatePending { + t.Fatalf("new row state = %q, want pending by default", u.State) + } + if _, err := repo.ClaimState(ctx, u.ID, UnmatchedStatePending, UnmatchedStateAdopting); err != nil { + t.Fatal(err) + } + if _, err := repo.CompleteAdoption(ctx, u.ID, AdoptionRecord{BookID: book.ID, CreatedBookID: book.ID, CreatedAuthorID: author.ID}); err != nil { + t.Fatal(err) + } + if err := books.Delete(ctx, book.ID); err != nil { + t.Fatalf("delete a book an adoption points at: %v", err) + } + if got := unitByPath(t, database, repo, "/lib/A/x.epub"); got.BookID != 0 || got.CreatedBookID != 0 { + t.Fatalf("book references after delete = %+v, want cleared", got) + } +} + +// BenchmarkReconcileScan is T6: one full reconcile at the scan's unit cap. +func BenchmarkReconcileScan(b *testing.B) { + ctx := context.Background() + units := make([]UnmatchedUnitScan, 20000) + for i := range units { + units[i] = scanUnit(fmt.Sprintf("/lib/Author %03d/Book %05d/book.epub", i%400, i)) + units[i].Candidates = []UnmatchedCandidate{{BookID: int64(i), Score: 0.7}} + } + _, repo := openUnmatchedRepo(b) + b.ResetTimer() + for b.Loop() { + if _, err := repo.ReconcileScan(ctx, units, ReconcileScanOptions{}); err != nil { + b.Fatal(err) + } + } +} From 858685324e60065a93a927700a72e25fbf0a945b Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 01:50:40 -0300 Subject: [PATCH 04/18] feat(scanner): group unmatched files into books and store them as units (#2547) The library scan used to put its first 1000 unmatched files into the lastScan blob, one entry per file, so a 193 track audiobook filled 193 rows and a large library lost everything past the cap. The scan now groups unmatched files into book units (audio by folder, disc folders by the book folder above, same stem ebooks together), ranks up to three suggestions per unit from the catalogue already in memory with the reconcile's own title measure, and stores them through WithUnmatchedUnits. Bounds are 50,000 files and 20,000 units with a truncated flag. Only regular files whose folder resolves inside a scanned root become rows (S10); size and mode come from the walk. The blob keeps its counters and the #1436 invariant, gains unmatched_units, ignored_units and units_truncated, and writes unmatched_files as an empty list for one release. The disc folder regex, AllDiscFolders and SameStem move from internal/api/scan_walk.go into importer so both scans share one definition. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- internal/api/scan_walk.go | 72 +--- internal/importer/scanner.go | 79 ++-- internal/importer/scanner_claim_rank_test.go | 4 +- .../importer/scanner_unmatched_blob_test.go | 96 +++++ .../importer/scanner_unmatched_reason_test.go | 54 ++- internal/importer/scanner_unmatched_test.go | 1 + internal/importer/unit_boundaries.go | 85 ++++ internal/importer/unmatched_units.go | 367 ++++++++++++++++++ internal/importer/unmatched_units_test.go | 353 +++++++++++++++++ 9 files changed, 964 insertions(+), 147 deletions(-) create mode 100644 internal/importer/scanner_unmatched_blob_test.go create mode 100644 internal/importer/unit_boundaries.go create mode 100644 internal/importer/unmatched_units.go create mode 100644 internal/importer/unmatched_units_test.go diff --git a/internal/api/scan_walk.go b/internal/api/scan_walk.go index 5537df7e4..01844c44a 100644 --- a/internal/api/scan_walk.go +++ b/internal/api/scan_walk.go @@ -3,9 +3,7 @@ package api import ( "os" "path/filepath" - "regexp" "sort" - "strings" "github.com/vavallee/bindery/internal/importer" ) @@ -19,13 +17,6 @@ type scanUnit struct { isDir bool } -// discFolderRe matches the disc/part subfolder names ("CD1", "Disc 2", -// "Part 03", "Vol. 1", or a bare "1"/"02") that split ONE audiobook across -// several directories. When every subdirectory of a folder looks like a disc, -// the folder is a single multi-disc audiobook rather than a shelf of separate -// books. -var discFolderRe = regexp.MustCompile(`(?i)^(cd|dis[ck]|part|pt|vol|volume|book|chapter|ch)\s*[._-]?\s*\d+$|^\d{1,2}$`) - // walkUnitLimits bounds the recursive enumeration so pointing the scan at a // pathological tree can't stall the request or allocate without bound. const ( @@ -101,12 +92,12 @@ func enumerateImportUnits(root string, limit int) (units []scanUnit, truncated b return } // Every subdir a disc folder → one multi-disc audiobook. - if len(subdirs) > 0 && allDiscFolders(subdirs) { + if len(subdirs) > 0 && importer.AllDiscFolders(subdirs) { emit(dir, true) return } // A leaf folder of same-named ebooks → one book in several formats. - if len(subdirs) == 0 && len(ebookFiles) >= 2 && sameStem(ebookFiles) { + if len(subdirs) == 0 && len(ebookFiles) >= 2 && importer.SameStem(ebookFiles) { emit(dir, true) return } @@ -139,62 +130,3 @@ func enumerateImportUnits(root string, limit int) (units []scanUnit, truncated b walk(root, 0, true) return units, truncated } - -// allDiscFolders reports whether every directory in dirs is a disc/part folder -// (by name) that actually holds audio somewhere beneath it. Both conditions are -// required so a shelf of audiobook folders with numeric-ish names isn't collapsed -// into one book, and an empty "CD1" placeholder doesn't fake a multi-disc set. -func allDiscFolders(dirs []string) bool { - for _, d := range dirs { - if !discFolderRe.MatchString(filepath.Base(d)) { - return false - } - if !dirSubtreeHasAudio(d) { - return false - } - } - return len(dirs) > 0 -} - -// dirSubtreeHasAudio reports whether dir contains at least one audio file within -// a bounded number of entries, so a deep tree can't stall the disc-folder check. -func dirSubtreeHasAudio(dir string) bool { - const limit = 2000 - count := 0 - found := false - _ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { - if err != nil { - return nil - } - count++ - if count > limit { - return filepath.SkipAll - } - if !d.IsDir() && importer.IsAudioFile(p) { - found = true - return filepath.SkipAll - } - return nil - }) - return found -} - -// sameStem reports whether every file shares one normalised base name (the file -// name without its extension, lower-cased). Title.epub and Title.mobi share the -// stem "title"; Dune.epub and Foundation.epub do not. -func sameStem(files []string) bool { - stem := func(p string) string { - b := filepath.Base(p) - return strings.ToLower(strings.TrimSuffix(b, filepath.Ext(b))) - } - if len(files) == 0 { - return false - } - first := stem(files[0]) - for _, f := range files[1:] { - if stem(f) != first { - return false - } - } - return true -} diff --git a/internal/importer/scanner.go b/internal/importer/scanner.go index f81c86b27..9acbc1355 100644 --- a/internal/importer/scanner.go +++ b/internal/importer/scanner.go @@ -101,6 +101,9 @@ type Scanner struct { absLib absNotifier absLibraryIDsFn func() []string notif eventNotifier + // unmatchedUnits stores the books a library scan could not match, for + // library adoption. Nil stores nothing (see WithUnmatchedUnits). + unmatchedUnits UnmatchedUnitStore // jobs, when set, tracks the detached scan goroutine launched by StartScan // so process shutdown can cancel and drain it before the database closes @@ -2852,9 +2855,13 @@ func (s *Scanner) scanLibrary(ctx context.Context) { return } + scanStartedAt := time.Now() + // walkDir appends all book files found under root to foundFiles, tracking // which root each file belongs to so the author-inference fallback can - // strip the correct prefix. + // strip the correct prefix. The size and mode the walk already has are + // kept for the unmatched units (P2). + walked := make(map[string]walkedFile) walkDir := func(root string) []string { var files []string if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { @@ -2863,6 +2870,7 @@ func (s *Scanner) scanLibrary(ctx context.Context) { } if IsBookFile(path) { files = append(files, path) + walked[path] = walkedFile{size: info.Size(), mode: info.Mode()} } return nil }); err != nil { @@ -2881,7 +2889,7 @@ func (s *Scanner) scanLibrary(ctx context.Context) { slog.Info("library scan found files", "paths", []string{s.libraryDir, s.audiobookDir}, "count", len(foundFiles)) if len(foundFiles) == 0 { - s.writeScanResult(ctx, len(foundFiles), 0, 0, 0, 0, nil) + s.writeScanResult(ctx, len(foundFiles), 0, 0, 0, 0, s.unmatchedUnitCounts(ctx, false)) return } @@ -3130,7 +3138,7 @@ func (s *Scanner) scanLibrary(ctx context.Context) { return set, parsedAuthor } - var unmatchedFiles []unmatchedFile + var unmatchedFiles unmatchedCollector var reconciled, unmatched, alreadyTracked, tagReadFailed int // tryReconcileTitle attempts to reconcile path to the given wanted book via @@ -3244,7 +3252,7 @@ func (s *Scanner) scanLibrary(ctx context.Context) { // every epub sitting next to an attached audiobook from the scan // (#1957) — the mirror image of the one-format-per-pass claim below. if trackedPaths[cleanPath] || - (detectedFmt == models.MediaTypeAudiobook && trackedPaths[filepath.Clean(filepath.Dir(cleanPath))]) { + (detectedFmt == models.MediaTypeAudiobook && audioTrackedByFolder(trackedPaths, cleanPath)) { alreadyTracked++ continue } @@ -3460,15 +3468,10 @@ func (s *Scanner) scanLibrary(ctx context.Context) { slog.Debug("library scan: unmatched file", "path", path, "parsedTitle", parsed.Title, "parsedAuthor", parsed.Author, "matchAuthor", matchAuthor, "reason", reason) unmatched++ - // Collect up to 1000 unmatched entries for UI display - if len(unmatchedFiles) < 1000 { - unmatchedFiles = append(unmatchedFiles, unmatchedFile{ - Path: path, - ParsedTitle: parsed.Title, - ParsedAuthor: parsed.Author, - Reason: reason, - }) - } + unmatchedFiles.add(unmatchedScanFile{ + path: path, format: detectedFmt, size: walked[path].size, mode: walked[path].mode, + title: parsed.Title, author: parsed.Author, layoutAuthor: layoutAuthor, reason: reason, + }) } } @@ -3488,7 +3491,15 @@ func (s *Scanner) scanLibrary(ctx context.Context) { slog.Info("library scan complete", "paths", scanRoots, "bookFiles", len(foundFiles), "reconciled", reconciled, "unmatched", unmatched, "tagReadFailed", tagReadFailed) - s.writeScanResult(ctx, len(foundFiles), reconciled, unmatched, alreadyTracked, tagReadFailed, unmatchedFiles) + // Suggestions come from the catalogue already in memory, ranked once per + // unit rather than per file. + units := s.recordUnmatchedUnits(ctx, &unmatchedFiles, scanRoots, scanStartedAt, + func(title, author, layoutAuthor string) []db.UnmatchedCandidate { + authorSet, _ := resolveAuthors(author, layoutAuthor) + return rankCandidates(normalizeTitle(title), wantedBooks, booksByAuthor, authorSet) + }) + + s.writeScanResult(ctx, len(foundFiles), reconciled, unmatched, alreadyTracked, tagReadFailed, units) } // refreshStaleRenderedPaths re-derives the file path shown for books that @@ -3640,22 +3651,12 @@ const ( unmatchedReasonNoTitleParsed = "no_title_parsed" ) -// unmatchedFile represents a file that could not be reconciled during library scan. -type unmatchedFile struct { - Path string `json:"path"` - ParsedTitle string `json:"parsed_title"` - ParsedAuthor string `json:"parsed_author"` - // Reason is one of the unmatchedReason* constants. Omitted when empty so - // results written before this field existed keep parsing unchanged. - Reason string `json:"reason,omitempty"` -} - // writeScanError persists a failed-scan result so the UI reflects the failure // instead of a stale prior scan (#965). It reuses the same "library.lastScan" // shape as a normal scan — zero counts plus a non-empty scan_error message the // frontend renders the same way as the other scan-outcome warnings. func (s *Scanner) writeScanError(ctx context.Context, message string) { - s.writeScanResultWithError(ctx, 0, 0, 0, 0, 0, nil, message) + s.writeScanResultWithError(ctx, 0, 0, 0, 0, 0, s.unmatchedUnitCounts(ctx, false), message) } // writeScanResult persists the scan summary to the settings table under @@ -3664,32 +3665,18 @@ func (s *Scanner) writeScanError(ctx context.Context, message string) { // api.SettingLibraryLastScan on the read side, where the settings endpoints // reserve it for admins because the blob carries absolute paths (#2361). Keep // the two spellings in sync. -func (s *Scanner) writeScanResult(ctx context.Context, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed int, unmatchedFiles []unmatchedFile) { - s.writeScanResultWithError(ctx, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed, unmatchedFiles, "") +func (s *Scanner) writeScanResult(ctx context.Context, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed int, units unitCounts) { + s.writeScanResultWithError(ctx, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed, units, "") } // writeScanResultWithError is the shared writer for both successful scans and // early-return failures. scanError is empty for a normal scan and a // user-facing message when the scan could not complete (#965). -func (s *Scanner) writeScanResultWithError(ctx context.Context, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed int, unmatchedFiles []unmatchedFile, scanError string) { +func (s *Scanner) writeScanResultWithError(ctx context.Context, filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed int, units unitCounts, scanError string) { if s.settings == nil { return } - // Marshal unmatched files to JSON - var unmatchedJSON string - if len(unmatchedFiles) > 0 { - bytes, err := json.Marshal(unmatchedFiles) - if err != nil { - slog.Warn("library scan: failed to marshal unmatched files", "error", err) - unmatchedJSON = "[]" - } else { - unmatchedJSON = string(bytes) - } - } else { - unmatchedJSON = "[]" - } - // Surface the resolved roots that were actually walked so the UI can tell // the user WHICH paths produced the file count. The scanner unions the // audiobook root with the library root only when it differs (see @@ -3719,11 +3706,15 @@ func (s *Scanner) writeScanResultWithError(ctx context.Context, filesFound, reco } } + // The unmatched files now live in unmatched_units, one row per book, and + // the blob carries only their counts. unmatched_files stays as an empty + // list so a web bundle cached from before library adoption still parses + // the result. Added 2026-09 for v1.37; remove it in the release after. payload := fmt.Sprintf( - `{"ran_at":%q,"files_found":%d,"reconciled":%d,"unmatched":%d,"already_tracked":%d,"tag_read_failed":%d,"unmatched_files":%s,"library_dir":%q,"audiobook_dir":%q,"scanned_paths":%s,"no_files_found":%t,"scan_error":%s}`, + `{"ran_at":%q,"files_found":%d,"reconciled":%d,"unmatched":%d,"already_tracked":%d,"tag_read_failed":%d,"unmatched_files":[],"unmatched_units":%d,"ignored_units":%d,"units_truncated":%t,"library_dir":%q,"audiobook_dir":%q,"scanned_paths":%s,"no_files_found":%t,"scan_error":%s}`, time.Now().UTC().Format(time.RFC3339), filesFound, reconciled, unmatched, alreadyTracked, tagReadFailed, - unmatchedJSON, + units.pending, units.ignored, units.truncated, s.libraryDir, s.audiobookDir, pathsJSON, noFilesFound, scanErrorJSON, ) if err := s.settings.Set(ctx, "library.lastScan", payload); err != nil { diff --git a/internal/importer/scanner_claim_rank_test.go b/internal/importer/scanner_claim_rank_test.go index b3cbd905a..11fe67b38 100644 --- a/internal/importer/scanner_claim_rank_test.go +++ b/internal/importer/scanner_claim_rank_test.go @@ -137,8 +137,8 @@ func TestScanLibrary_EbookContainerOutranksTextSidecar(t *testing.T) { // The passed-over sidecar is the container's companion, not an // orphan the user has to go and fix, so it must not show up in the // Unmatched list either. - for _, f := range readUnmatchedFiles(t, ctx, settings) { - if filepath.Clean(f.Path) == filepath.Clean(sidecar) { + for _, f := range readUnmatchedFiles(t, ctx, s) { + if filepath.Clean(f.UnitPath) == filepath.Clean(sidecar) { t.Errorf("sidecar reported unmatched with reason %q; it is the epub's companion", f.Reason) } } diff --git a/internal/importer/scanner_unmatched_blob_test.go b/internal/importer/scanner_unmatched_blob_test.go new file mode 100644 index 000000000..c35bf41f5 --- /dev/null +++ b/internal/importer/scanner_unmatched_blob_test.go @@ -0,0 +1,96 @@ +package importer + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/vavallee/bindery/internal/db" +) + +// This file reads only the library.lastScan blob, so it builds and runs +// against the tree from before library adoption too. That is what records +// the fail before evidence for #2547 and the dropped 1000 file cap. + +type unmatchedBlob struct { + FilesFound int `json:"files_found"` + Reconciled int `json:"reconciled"` + Unmatched int `json:"unmatched"` + AlreadyTracked int `json:"already_tracked"` + UnmatchedFiles []json.RawMessage `json:"unmatched_files"` + UnmatchedUnits *int `json:"unmatched_units"` + UnitsTruncated bool `json:"units_truncated"` +} + +func readUnmatchedBlob(t *testing.T, ctx context.Context, settings *db.SettingsRepo) unmatchedBlob { + t.Helper() + setting, err := settings.Get(ctx, "library.lastScan") + if err != nil || setting == nil { + t.Fatalf("library.lastScan: %v %v", setting, err) + } + var b unmatchedBlob + if err := json.Unmarshal([]byte(setting.Value), &b); err != nil { + t.Fatalf("unmarshal scan result: %v", err) + } + return b +} + +// TestScanLibrary_TrackFolderIsOneUnmatchedBook is #2547: a 193 track +// audiobook nobody has in the library is one book to decide about, not 193 +// rows. The file counters keep their #1436 invariant (files_found equals +// reconciled + unmatched + already_tracked) because they count files. +func TestScanLibrary_TrackFolderIsOneUnmatchedBook(t *testing.T) { + s, _, _, _, settings, libraryDir, ctx := unmatchedFixture(t) + dir := filepath.Join(libraryDir, "Brandon Sanderson", "Rhythm of War") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for i := 1; i <= 193; i++ { + track := filepath.Join(dir, fmt.Sprintf("%03d - Rhythm of War.mp3", i)) + if err := os.WriteFile(track, buildID3v23(fmt.Sprintf("%03d-193", i), "Brandon Sanderson", ""), 0o644); err != nil { + t.Fatal(err) + } + } + + s.ScanLibrary(ctx) + + b := readUnmatchedBlob(t, ctx, settings) + if b.FilesFound != 193 || b.Unmatched != 193 || b.FilesFound != b.Reconciled+b.Unmatched+b.AlreadyTracked { + t.Errorf("counters = %+v, want 193 files, all unmatched, invariant intact", b) + } + if len(b.UnmatchedFiles) != 0 { + t.Errorf("unmatched_files has %d per file entries, want none: the list is units now", len(b.UnmatchedFiles)) + } + if b.UnmatchedUnits == nil || *b.UnmatchedUnits != 1 { + t.Errorf("unmatched_units = %v, want 1 book for the 193 tracks", b.UnmatchedUnits) + } +} + +// TestScanLibrary_UnmatchedListHasNoThousandCap: the old list stopped at 1000 +// files and silently dropped the rest. 1200 unmatched books are 1200 units. +func TestScanLibrary_UnmatchedListHasNoThousandCap(t *testing.T) { + s, _, _, _, settings, libraryDir, ctx := unmatchedFixture(t) + dir := filepath.Join(libraryDir, "Unknown Author") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for i := range 1200 { + if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("Book Number %04d.txt", i)), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + s.ScanLibrary(ctx) + + b := readUnmatchedBlob(t, ctx, settings) + if b.Unmatched != 1200 { + t.Fatalf("unmatched = %d, want 1200", b.Unmatched) + } + if b.UnmatchedUnits == nil || *b.UnmatchedUnits != 1200 || b.UnitsTruncated { + t.Errorf("unmatched_units = %v (truncated %v), want all 1200 stored; unmatched_files held %d", + b.UnmatchedUnits, b.UnitsTruncated, len(b.UnmatchedFiles)) + } +} diff --git a/internal/importer/scanner_unmatched_reason_test.go b/internal/importer/scanner_unmatched_reason_test.go index b891354fd..f6cc4c618 100644 --- a/internal/importer/scanner_unmatched_reason_test.go +++ b/internal/importer/scanner_unmatched_reason_test.go @@ -2,7 +2,6 @@ package importer import ( "context" - "encoding/json" "os" "path/filepath" "testing" @@ -11,32 +10,25 @@ import ( "github.com/vavallee/bindery/internal/models" ) -// unmatchedReasonPayload reads back just the unmatched entries of the persisted -// scan result, including the #1958 reason field. -type unmatchedReasonPayload struct { - Unmatched int `json:"unmatched"` - UnmatchedFiles []unmatchedFile `json:"unmatched_files"` -} - -func readUnmatchedFiles(t *testing.T, ctx context.Context, settings *db.SettingsRepo) []unmatchedFile { +// readUnmatchedFiles reads back the unmatched units the scan stored, with the +// #1958 reason each carries. Each test here lays one file per book, so a unit +// is one file. +func readUnmatchedFiles(t *testing.T, ctx context.Context, s *Scanner) []db.UnmatchedUnit { t.Helper() - setting, err := settings.Get(ctx, "library.lastScan") - if err != nil { - t.Fatalf("get library.lastScan: %v", err) + repo, ok := s.unmatchedUnits.(*db.UnmatchedUnitRepo) + if !ok { + t.Fatal("scanner has no unmatched unit repo wired") } - if setting == nil { - t.Fatal("expected library.lastScan to be persisted, got nil") - } - var p unmatchedReasonPayload - if err := json.Unmarshal([]byte(setting.Value), &p); err != nil { - t.Fatalf("unmarshal scan result %q: %v", setting.Value, err) + units, _, err := repo.List(ctx, db.UnmatchedListQuery{Limit: 250}) + if err != nil { + t.Fatalf("list unmatched units: %v", err) } - return p.UnmatchedFiles + return units } // scanOneUnmatched lays a single epub at rel under the library dir, scans, and // returns the one unmatched entry the scan must have produced. -func scanOneUnmatched(t *testing.T, s *Scanner, settings *db.SettingsRepo, ctx context.Context, libraryDir string, rel ...string) unmatchedFile { +func scanOneUnmatched(t *testing.T, s *Scanner, ctx context.Context, libraryDir string, rel ...string) db.UnmatchedUnit { t.Helper() path := filepath.Join(append([]string{libraryDir}, rel...)...) if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { @@ -46,7 +38,7 @@ func scanOneUnmatched(t *testing.T, s *Scanner, settings *db.SettingsRepo, ctx c s.ScanLibrary(ctx) - files := readUnmatchedFiles(t, ctx, settings) + files := readUnmatchedFiles(t, ctx, s) if len(files) != 1 { t.Fatalf("expected exactly one unmatched file, got %d (%+v)", len(files), files) } @@ -59,7 +51,7 @@ func scanOneUnmatched(t *testing.T, s *Scanner, settings *db.SettingsRepo, ctx c // cannot help — there is no such author to refresh — and it pointed away from // the evidence (the parsed author) for two weeks. func TestScanLibrary_UnmatchedReasonAuthorNotInLibrary(t *testing.T) { - s, _, books, authors, settings, libraryDir, ctx := unmatchedFixture(t) + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) author := &models.Author{ForeignID: "ol:weir", Name: "Andy Weir", SortName: "Weir, Andy", Monitored: true, MetadataProvider: "openlibrary"} if err := authors.Create(ctx, author); err != nil { @@ -74,7 +66,7 @@ func TestScanLibrary_UnmatchedReasonAuthorNotInLibrary(t *testing.T) { t.Fatal(err) } - got := scanOneUnmatched(t, s, settings, ctx, libraryDir, "Becky Chambers", "A Psalm for the Wild-Built", "A Psalm for the Wild-Built.epub") + got := scanOneUnmatched(t, s, ctx, libraryDir, "Becky Chambers", "A Psalm for the Wild-Built", "A Psalm for the Wild-Built.epub") if got.Reason != unmatchedReasonAuthorNotInLibrary { t.Errorf("reason = %q, want %q", got.Reason, unmatchedReasonAuthorNotInLibrary) } @@ -87,14 +79,14 @@ func TestScanLibrary_UnmatchedReasonAuthorNotInLibrary(t *testing.T) { // pointed at the case it was written for (#875): the author is in the library // but has no book the scan can attach a file to. func TestScanLibrary_UnmatchedReasonNoCandidateBooks(t *testing.T) { - s, _, _, authors, settings, libraryDir, ctx := unmatchedFixture(t) + s, _, _, authors, _, libraryDir, ctx := unmatchedFixture(t) author := &models.Author{ForeignID: "ol:weir", Name: "Andy Weir", SortName: "Weir, Andy", Monitored: true, MetadataProvider: "openlibrary"} if err := authors.Create(ctx, author); err != nil { t.Fatal(err) } - got := scanOneUnmatched(t, s, settings, ctx, libraryDir, "Andy Weir", "Project Hail Mary", "Project Hail Mary.epub") + got := scanOneUnmatched(t, s, ctx, libraryDir, "Andy Weir", "Project Hail Mary", "Project Hail Mary.epub") if got.Reason != unmatchedReasonNoCandidateBooks { t.Errorf("reason = %q, want %q", got.Reason, unmatchedReasonNoCandidateBooks) } @@ -103,7 +95,7 @@ func TestScanLibrary_UnmatchedReasonNoCandidateBooks(t *testing.T) { // TestScanLibrary_UnmatchedReasonNoTitleMatch covers the third case: the author // matched and has wanted books, but none of their titles is this file. func TestScanLibrary_UnmatchedReasonNoTitleMatch(t *testing.T) { - s, _, books, authors, settings, libraryDir, ctx := unmatchedFixture(t) + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) author := &models.Author{ForeignID: "ol:weir", Name: "Andy Weir", SortName: "Weir, Andy", Monitored: true, MetadataProvider: "openlibrary"} if err := authors.Create(ctx, author); err != nil { @@ -118,7 +110,7 @@ func TestScanLibrary_UnmatchedReasonNoTitleMatch(t *testing.T) { t.Fatal(err) } - got := scanOneUnmatched(t, s, settings, ctx, libraryDir, "Andy Weir", "The Martian", "The Martian.epub") + got := scanOneUnmatched(t, s, ctx, libraryDir, "Andy Weir", "The Martian", "The Martian.epub") if got.Reason != unmatchedReasonNoTitleMatch { t.Errorf("reason = %q, want %q", got.Reason, unmatchedReasonNoTitleMatch) } @@ -130,7 +122,7 @@ func TestScanLibrary_UnmatchedReasonNoTitleMatch(t *testing.T) { // about a file that had no title to match — the fix is to rename the file, and // the reason has to say that. func TestScanLibrary_UnmatchedReasonNoTitleParsed(t *testing.T) { - s, _, books, authors, settings, libraryDir, ctx := unmatchedFixture(t) + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) author := &models.Author{ForeignID: "ol:weir", Name: "Andy Weir", SortName: "Weir, Andy", Monitored: true, MetadataProvider: "openlibrary"} if err := authors.Create(ctx, author); err != nil { @@ -147,7 +139,7 @@ func TestScanLibrary_UnmatchedReasonNoTitleParsed(t *testing.T) { // A name that is nothing but a stripped parenthetical year parses to an // empty title, which is exactly the shape a release dump leaves behind. - got := scanOneUnmatched(t, s, settings, ctx, libraryDir, "Andy Weir", "(2021).epub") + got := scanOneUnmatched(t, s, ctx, libraryDir, "Andy Weir", "(2021).epub") if got.Reason != unmatchedReasonNoTitleParsed { t.Errorf("reason = %q, want %q", got.Reason, unmatchedReasonNoTitleParsed) } @@ -159,7 +151,7 @@ func TestScanLibrary_UnmatchedReasonNoTitleParsed(t *testing.T) { // match. The reason has to be computed from the same resolution the matcher // used, so this file is reported as a title miss. func TestScanLibrary_UnmatchedReasonUsesTheAuthorFallback(t *testing.T) { - s, _, books, authors, settings, libraryDir, ctx := unmatchedFixture(t) + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) author := &models.Author{ForeignID: "ol:enrigue", Name: "Álvaro Enrigue", SortName: "Enrigue, Álvaro", Monitored: true, MetadataProvider: "openlibrary"} if err := authors.Create(ctx, author); err != nil { @@ -185,7 +177,7 @@ func TestScanLibrary_UnmatchedReasonUsesTheAuthorFallback(t *testing.T) { s.ScanLibrary(ctx) - files := readUnmatchedFiles(t, ctx, settings) + files := readUnmatchedFiles(t, ctx, s) if len(files) != 1 { t.Fatalf("expected exactly one unmatched file, got %d (%+v)", len(files), files) } diff --git a/internal/importer/scanner_unmatched_test.go b/internal/importer/scanner_unmatched_test.go index 8069f690b..609ea8989 100644 --- a/internal/importer/scanner_unmatched_test.go +++ b/internal/importer/scanner_unmatched_test.go @@ -62,6 +62,7 @@ func unmatchedFixture(t *testing.T) (s *Scanner, downloads *db.DownloadRepo, boo libraryDir = t.TempDir() s = NewScanner(downloads, db.NewDownloadClientRepo(database), books, authors, db.NewHistoryRepo(database), libraryDir, "", "", "", "") s.WithSettings(settings) + s.WithUnmatchedUnits(db.NewUnmatchedUnitRepo(database)) if err := settings.Set(ctx, "import.mode", "copy"); err != nil { t.Fatal(err) } diff --git a/internal/importer/unit_boundaries.go b/internal/importer/unit_boundaries.go new file mode 100644 index 000000000..634967e2a --- /dev/null +++ b/internal/importer/unit_boundaries.go @@ -0,0 +1,85 @@ +package importer + +import ( + "os" + "path/filepath" + "regexp" + "strings" +) + +// The helpers below decide where one book ends and the next begins. Both the +// folder import scan (internal/api/scan_walk.go) and the library scan's +// unmatched grouping (unmatched_units.go) use them, so they live here once. + +// discFolderRe matches the disc/part subfolder names ("CD1", "Disc 2", +// "Part 03", "Vol. 1", or a bare "1"/"02") that split ONE audiobook across +// several directories. When every subdirectory of a folder looks like a disc, +// the folder is a single multi-disc audiobook rather than a shelf of separate +// books. +var discFolderRe = regexp.MustCompile(`(?i)^(cd|dis[ck]|part|pt|vol|volume|book|chapter|ch)\s*[._-]?\s*\d+$|^\d{1,2}$`) + +// IsDiscFolderName reports whether a directory's base name looks like one disc +// or part of a multi-disc audiobook. The name alone; see AllDiscFolders for +// the check that also requires audio beneath it. +func IsDiscFolderName(name string) bool { + return discFolderRe.MatchString(name) +} + +// AllDiscFolders reports whether every directory in dirs is a disc/part folder +// (by name) that actually holds audio somewhere beneath it. Both conditions are +// required so a shelf of audiobook folders with numeric-ish names isn't collapsed +// into one book, and an empty "CD1" placeholder doesn't fake a multi-disc set. +func AllDiscFolders(dirs []string) bool { + for _, d := range dirs { + if !discFolderRe.MatchString(filepath.Base(d)) { + return false + } + if !dirSubtreeHasAudio(d) { + return false + } + } + return len(dirs) > 0 +} + +// dirSubtreeHasAudio reports whether dir contains at least one audio file within +// a bounded number of entries, so a deep tree can't stall the disc-folder check. +func dirSubtreeHasAudio(dir string) bool { + const limit = 2000 + count := 0 + found := false + _ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + count++ + if count > limit { + return filepath.SkipAll + } + if !d.IsDir() && IsAudioFile(p) { + found = true + return filepath.SkipAll + } + return nil + }) + return found +} + +// SameStem reports whether every file shares one normalised base name (the file +// name without its extension, lower-cased). Title.epub and Title.mobi share the +// stem "title"; Dune.epub and Foundation.epub do not. +func SameStem(files []string) bool { + stem := func(p string) string { + b := filepath.Base(p) + return strings.ToLower(strings.TrimSuffix(b, filepath.Ext(b))) + } + if len(files) == 0 { + return false + } + first := stem(files[0]) + for _, f := range files[1:] { + if stem(f) != first { + return false + } + } + return true +} diff --git a/internal/importer/unmatched_units.go b/internal/importer/unmatched_units.go new file mode 100644 index 000000000..ecdd1e7bc --- /dev/null +++ b/internal/importer/unmatched_units.go @@ -0,0 +1,367 @@ +package importer + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "time" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/models" + "github.com/vavallee/bindery/internal/textutil" +) + +// Library adoption's side of the scan: the files the reconcile loop could not +// match are grouped into books and stored as unmatched_units rows, where an +// admin can adopt, ignore or leave them (see internal/api/adoption.go). + +const ( + // maxUnmatchedFiles bounds how many unmatched files one scan groups. The + // old blob list stopped at 1000 files, which a single large audiobook + // could fill on its own (#2547). + maxUnmatchedFiles = 50000 + // maxUnmatchedUnits bounds how many rows one scan stores. + maxUnmatchedUnits = 20000 + // candidateThreshold is the lowest title similarity offered as a + // suggestion. The reconcile itself needs 0.85; below that a person decides. + candidateThreshold = 0.60 + // maxCandidates is how many suggestions a unit keeps. + maxCandidates = 3 +) + +// UnmatchedUnitStore is where the scan records its unmatched units. +type UnmatchedUnitStore interface { + ReconcileScan(ctx context.Context, units []db.UnmatchedUnitScan, opts db.ReconcileScanOptions) (db.ReconcileScanResult, error) + Summary(ctx context.Context) (db.UnmatchedSummary, error) +} + +// WithUnmatchedUnits attaches the store the library scan writes unmatched +// units to. Nil leaves the scan counting unmatched files and storing nothing. +func (s *Scanner) WithUnmatchedUnits(store UnmatchedUnitStore) *Scanner { + s.unmatchedUnits = store + return s +} + +// walkedFile is what the library walk already knew about a file (P2): its +// size and mode come from the os.FileInfo filepath.Walk hands over, so +// grouping costs no extra stat. +type walkedFile struct { + size int64 + mode os.FileMode +} + +// unmatchedScanFile is one file the reconcile loop left unmatched, with what +// the loop parsed out of it. +type unmatchedScanFile struct { + path string + format string + size int64 + mode os.FileMode + title string + author string + layoutAuthor string + reason string +} + +// unmatchedCollector gathers unmatched files up to maxUnmatchedFiles. +type unmatchedCollector struct { + files []unmatchedScanFile + truncated bool +} + +func (c *unmatchedCollector) add(f unmatchedScanFile) { + if len(c.files) >= maxUnmatchedFiles { + c.truncated = true + return + } + c.files = append(c.files, f) +} + +// unmatchedGroup is one unit before candidates are ranked: the stored shape +// plus the member whose parse speaks for the unit. +type unmatchedGroup struct { + unit db.UnmatchedUnitScan + rep unmatchedScanFile +} + +// scanRootFor returns the longest root that contains path, or "". +func scanRootFor(path string, roots []string) string { + best := "" + for _, r := range roots { + if r != "" && pathUnderDir(path, r) && len(r) > len(best) { + best = r + } + } + return best +} + +// unitKeyFor decides which unit a file belongs to (#2547). The unit is a book, +// not a file: +// +// - audio groups by its folder, so 193 tracks are one row; +// - audio in a disc folder (CD1, Disc 2) groups by the folder above it; +// - loose audio directly in a library root has no book folder, so each file +// stands alone, as in the folder import scan; +// - ebooks group by folder and file stem, so Title.epub and Title.mobi are +// one book in two formats. +// +// It returns the grouping key, and for a folder unit the folder path. +func unitKeyFor(f unmatchedScanFile, root string) (key, folder string) { + parent := filepath.Dir(f.path) + if f.format == models.MediaTypeAudiobook { + if parent == root { + return "file\x00" + f.path, "" + } + if IsDiscFolderName(filepath.Base(parent)) && filepath.Dir(parent) != root && pathUnderDir(filepath.Dir(parent), root) { + parent = filepath.Dir(parent) + } + return "folder\x00" + parent, parent + } + base := filepath.Base(f.path) + return "stem\x00" + parent + "\x00" + strings.ToLower(strings.TrimSuffix(base, filepath.Ext(base))), "" +} + +// groupUnmatched turns unmatched files into book units. Every file lands in +// exactly one unit, and the units' file counts sum to the number of files +// given (TestGroupUnmatched_Property). Units come back ordered by path. More +// than maxUnmatchedUnits units is truncated to that many. +func groupUnmatched(files []unmatchedScanFile, roots []string) (groups []unmatchedGroup, truncated bool) { + type acc struct { + folder string + root string + members []unmatchedScanFile + } + byKey := make(map[string]*acc) + var keys []string + for _, f := range files { + root := scanRootFor(f.path, roots) + key, folder := unitKeyFor(f, root) + a := byKey[key] + if a == nil { + a = &acc{folder: folder, root: root} + byKey[key] = a + keys = append(keys, key) + } + a.members = append(a.members, f) + } + + groups = make([]unmatchedGroup, 0, len(keys)) + for _, key := range keys { + a := byKey[key] + slices.SortFunc(a.members, func(x, y unmatchedScanFile) int { return strings.Compare(x.path, y.path) }) + u := db.UnmatchedUnitScan{ + UnitPath: a.members[0].path, + UnitKind: db.UnmatchedKindFile, + Format: a.members[0].format, + FileCount: len(a.members), + RootPath: a.root, + } + if a.folder != "" { + u.UnitPath, u.UnitKind = a.folder, db.UnmatchedKindFolder + } + u.MemberPaths = make([]string, len(a.members)) + for i, m := range a.members { + u.MemberPaths[i] = m.path + u.SizeBytes += m.size + } + if a.root != "" { + if rel, err := filepath.Rel(a.root, u.UnitPath); err == nil { + u.RelPath = filepath.ToSlash(rel) + } + if rel, err := filepath.Rel(a.root, a.members[0].path); err == nil { + if parts := strings.Split(filepath.ToSlash(rel), "/"); len(parts) >= 2 { + u.AuthorFolder = parts[0] + } + } + } + rep := representative(a.members) + u.ParsedTitle, u.ParsedAuthor, u.Reason = rep.title, rep.author, rep.reason + groups = append(groups, unmatchedGroup{unit: u, rep: rep}) + } + sort.Slice(groups, func(i, j int) bool { return groups[i].unit.UnitPath < groups[j].unit.UnitPath }) + if len(groups) > maxUnmatchedUnits { + groups, truncated = groups[:maxUnmatchedUnits], true + } + return groups, truncated +} + +// representative picks the member whose parse speaks for the unit: the one +// carrying the title most members agree on. For an audiobook folder that is +// the folder's title rather than one track's chapter name. Members arrive +// sorted, so ties go to the first path. +func representative(members []unmatchedScanFile) unmatchedScanFile { + counts := make(map[string]int, len(members)) + best := members[0] + for _, m := range members { + counts[m.title]++ + if counts[m.title] > counts[best.title] { + best = m + } + } + return best +} + +// eligibleUnmatched keeps only the files that may become rows (S10): regular +// files, by the mode the walk read with Lstat, whose resolved folder lies in +// a scanned root. A symlink inside the library would otherwise become a +// tracked book file that the download and OPDS routes serve from wherever it +// points. Each folder is resolved once. +func eligibleUnmatched(files []unmatchedScanFile, roots []string) []unmatchedScanFile { + realRoots := make([]string, 0, len(roots)) + for _, r := range roots { + if r == "" { + continue + } + if rr, err := filepath.EvalSymlinks(r); err == nil { + realRoots = append(realRoots, rr) + } + } + resolved := make(map[string]bool) + out := files[:0:0] + for _, f := range files { + if !f.mode.IsRegular() { + slog.Debug("library scan: unmatched entry is not a regular file; not listing it", "path", f.path) + continue + } + dir := filepath.Dir(f.path) + ok, seen := resolved[dir] + if !seen { + if real, err := filepath.EvalSymlinks(dir); err == nil { + for _, r := range realRoots { + if pathUnderDir(real, r) { + ok = true + break + } + } + } + resolved[dir] = ok + } + if !ok { + slog.Debug("library scan: unmatched file resolves outside the library; not listing it", "path", f.path) + continue + } + out = append(out, f) + } + return out +} + +// rankCandidates returns up to maxCandidates catalogue books whose title +// scores at least candidateThreshold against normParsed, best first. It uses +// the reconcile's own measure (Jaro-Winkler over normalizeTitle) and its own +// candidate set: the books of the authors the file's author resolved to, or +// every reconcile candidate when the file named no author. In that last case +// titles of wildly different length are skipped, the same cheap gate the +// reconcile uses. No provider is asked anything. +func rankCandidates(normParsed string, wanted []scanBook, byAuthor map[int64][]int, authorSet map[int64]bool) []db.UnmatchedCandidate { + if normParsed == "" { + return nil + } + var out []db.UnmatchedCandidate + consider := func(sb *scanBook) { + score := textutil.JaroWinkler(sb.normTitle, normParsed) + if score < candidateThreshold { + return + } + out = append(out, db.UnmatchedCandidate{BookID: sb.book.ID, Score: score}) + } + if authorSet == nil { + for i := range wanted { + lo, hi := wanted[i].normLen, len(normParsed) + if lo > hi { + lo, hi = hi, lo + } + if lo == 0 || lo*4 < hi { + continue + } + consider(&wanted[i]) + } + } else { + ids := make([]int64, 0, len(authorSet)) + for id := range authorSet { + ids = append(ids, id) + } + slices.Sort(ids) + for _, id := range ids { + for _, idx := range byAuthor[id] { + consider(&wanted[idx]) + } + } + } + slices.SortStableFunc(out, func(a, b db.UnmatchedCandidate) int { + switch { + case a.Score > b.Score: + return -1 + case a.Score < b.Score: + return 1 + } + return 0 + }) + if len(out) > maxCandidates { + out = out[:maxCandidates] + } + return out +} + +// unitCounts is what the scan result blob reports about stored units. +type unitCounts struct { + pending int + ignored int + truncated bool +} + +// recordUnmatchedUnits groups a finished scan's unmatched files and stores +// them. candidatesFor ranks suggestions for one unit's representative parse. +func (s *Scanner) recordUnmatchedUnits(ctx context.Context, c *unmatchedCollector, roots []string, startedAt time.Time, + candidatesFor func(title, author, layoutAuthor string) []db.UnmatchedCandidate) unitCounts { + if s.unmatchedUnits == nil { + return unitCounts{} + } + groups, unitsTruncated := groupUnmatched(eligibleUnmatched(c.files, roots), roots) + units := make([]db.UnmatchedUnitScan, len(groups)) + for i, g := range groups { + units[i] = g.unit + units[i].Candidates = candidatesFor(g.rep.title, g.rep.author, g.rep.layoutAuthor) + } + truncated := c.truncated || unitsTruncated + res, err := s.unmatchedUnits.ReconcileScan(ctx, units, db.ReconcileScanOptions{ + StartedAt: startedAt, + SkipPurge: truncated, + }) + if err != nil { + slog.Warn("library scan: failed to store unmatched units", "error", err) + return s.unmatchedUnitCounts(ctx, truncated) + } + slog.Info("library scan: stored unmatched units", "units", res.Upserted, "removed", res.RemovedPending, + "purged", res.Purged, "pending", res.Pending, "ignored", res.Ignored, "truncated", truncated) + return unitCounts{pending: res.Pending, ignored: res.Ignored, truncated: truncated} +} + +// unmatchedUnitCounts reads the counts without changing any row, for a scan +// that found no files or failed: those must not delete anyone's decisions. +func (s *Scanner) unmatchedUnitCounts(ctx context.Context, truncated bool) unitCounts { + if s.unmatchedUnits == nil { + return unitCounts{} + } + sum, err := s.unmatchedUnits.Summary(ctx) + if err != nil { + slog.Warn("library scan: failed to count unmatched units", "error", err) + return unitCounts{truncated: truncated} + } + return unitCounts{pending: sum.Pending, ignored: sum.Ignored, truncated: truncated} +} + +// audioTrackedByFolder reports whether an audio file sits in a folder that is +// already tracked: its own folder, or, for a disc folder, the book folder +// above it. Adoption registers a disc set by its book folder, so without the +// second case the next scan would list every disc's tracks again. +func audioTrackedByFolder(trackedPaths map[string]bool, cleanPath string) bool { + parent := filepath.Dir(cleanPath) + if trackedPaths[parent] { + return true + } + return IsDiscFolderName(filepath.Base(parent)) && trackedPaths[filepath.Dir(parent)] +} diff --git a/internal/importer/unmatched_units_test.go b/internal/importer/unmatched_units_test.go new file mode 100644 index 000000000..dbc6ce568 --- /dev/null +++ b/internal/importer/unmatched_units_test.go @@ -0,0 +1,353 @@ +package importer + +import ( + "fmt" + "math/rand/v2" + "net/http" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/vavallee/bindery/internal/models" +) + +func ebookFile(path string) unmatchedScanFile { + return unmatchedScanFile{path: path, format: models.MediaTypeEbook, size: 100, mode: 0o644, title: filepath.Base(path)} +} + +func audioFile(path, title string) unmatchedScanFile { + return unmatchedScanFile{path: path, format: models.MediaTypeAudiobook, size: 1000, mode: 0o644, title: title} +} + +// TestGroupUnmatched_Table pins where one book ends and the next begins. +func TestGroupUnmatched_Table(t *testing.T) { + root := "/lib" + type want struct { + path, kind, format, folder, rel, title string + files int + } + tests := []struct { + name string + files []unmatchedScanFile + want []want + }{ + { + name: "audio tracks in a book folder are one folder unit", + files: []unmatchedScanFile{ + audioFile("/lib/Weir/Hail Mary/02.mp3", "Chapter 2"), + audioFile("/lib/Weir/Hail Mary/01.mp3", "Project Hail Mary"), + audioFile("/lib/Weir/Hail Mary/03.mp3", "Project Hail Mary"), + }, + want: []want{{"/lib/Weir/Hail Mary", "folder", "audiobook", "Weir", "Weir/Hail Mary", "Project Hail Mary", 3}}, + }, + { + name: "disc folders group under the book folder", + files: []unmatchedScanFile{ + audioFile("/lib/Weir/Artemis/CD1/01.mp3", "Artemis"), + audioFile("/lib/Weir/Artemis/Disc 2/01.mp3", "Artemis"), + }, + want: []want{{"/lib/Weir/Artemis", "folder", "audiobook", "Weir", "Weir/Artemis", "Artemis", 2}}, + }, + { + name: "a disc folder straight under the root is its own unit", + files: []unmatchedScanFile{ + audioFile("/lib/CD1/01.mp3", "Something"), + audioFile("/lib/CD1/02.mp3", "Something"), + }, + want: []want{{"/lib/CD1", "folder", "audiobook", "CD1", "CD1", "Something", 2}}, + }, + { + name: "loose audio at the root stands alone", + files: []unmatchedScanFile{ + audioFile("/lib/a.m4b", "A"), + audioFile("/lib/b.m4b", "B"), + }, + want: []want{ + {"/lib/a.m4b", "file", "audiobook", "", "a.m4b", "A", 1}, + {"/lib/b.m4b", "file", "audiobook", "", "b.m4b", "B", 1}, + }, + }, + { + name: "same stem ebooks are one book, other stems are separate", + files: []unmatchedScanFile{ + ebookFile("/lib/Herbert/Dune.mobi"), + ebookFile("/lib/Herbert/Dune.epub"), + ebookFile("/lib/Herbert/Children of Dune.epub"), + }, + want: []want{ + {"/lib/Herbert/Children of Dune.epub", "file", "ebook", "Herbert", "Herbert/Children of Dune.epub", "Children of Dune.epub", 1}, + {"/lib/Herbert/Dune.epub", "file", "ebook", "Herbert", "Herbert/Dune.epub", "Dune.epub", 2}, + }, + }, + { + name: "an ebook beside audio is a separate book unit", + files: []unmatchedScanFile{ + audioFile("/lib/X/Book/01.mp3", "Book"), + ebookFile("/lib/X/Book/Book.epub"), + }, + want: []want{ + {"/lib/X/Book", "folder", "audiobook", "X", "X/Book", "Book", 1}, + {"/lib/X/Book/Book.epub", "file", "ebook", "X", "X/Book/Book.epub", "Book.epub", 1}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + groups, truncated := groupUnmatched(tt.files, []string{root}) + if truncated { + t.Fatal("unexpected truncation") + } + if len(groups) != len(tt.want) { + t.Fatalf("got %d units, want %d: %+v", len(groups), len(tt.want), groups) + } + for i, w := range tt.want { + u := groups[i].unit + got := want{u.UnitPath, u.UnitKind, u.Format, u.AuthorFolder, u.RelPath, u.ParsedTitle, u.FileCount} + if got != w { + t.Errorf("unit %d = %+v, want %+v", i, got, w) + } + if len(u.MemberPaths) != u.FileCount || !slices.IsSorted(u.MemberPaths) { + t.Errorf("unit %d members %v do not match its file count", i, u.MemberPaths) + } + } + }) + } +} + +// TestGroupUnmatched_Property is T2: for random trees, every file lands in +// exactly one unit and the units' file counts and sizes sum to the input. +func TestGroupUnmatched_Property(t *testing.T) { + rng := rand.New(rand.NewPCG(2547, 1436)) + dirs := []string{"/lib", "/lib/A", "/lib/A/B1", "/lib/A/B1/CD1", "/lib/A/B1/CD2", "/lib/A/B2", "/lib/C", "/lib/C/Part 3", "/audio", "/audio/D/E"} + stems := []string{"one", "two", "One", "three"} + exts := []string{".epub", ".mobi", ".mp3", ".m4b", ".pdf"} + for trial := range 300 { + n := rng.IntN(60) + seen := map[string]bool{} + var files []unmatchedScanFile + var wantSize int64 + for range n { + p := filepath.Join(dirs[rng.IntN(len(dirs))], fmt.Sprintf("%s%d%s", stems[rng.IntN(len(stems))], rng.IntN(3), exts[rng.IntN(len(exts))])) + if seen[p] { + continue + } + seen[p] = true + f := ebookFile(p) + f.title = stems[rng.IntN(len(stems))] + if detectDownloadFormat([]string{p}) == models.MediaTypeAudiobook { + f.format = models.MediaTypeAudiobook + } + f.size = int64(rng.IntN(5000)) + wantSize += f.size + files = append(files, f) + } + groups, _ := groupUnmatched(files, []string{"/lib", "/audio"}) + placed := map[string]int{} + total := 0 + var size int64 + for _, g := range groups { + total += g.unit.FileCount + size += g.unit.SizeBytes + for _, m := range g.unit.MemberPaths { + placed[m]++ + } + } + if total != len(files) || size != wantSize { + t.Fatalf("trial %d: units hold %d files / %d bytes, input %d / %d", trial, total, size, len(files), wantSize) + } + for _, f := range files { + if placed[f.path] != 1 { + t.Fatalf("trial %d: %s placed %d times", trial, f.path, placed[f.path]) + } + } + } +} + +// TestEligibleUnmatched_SymlinkOutsideRoot is S10: a symlink inside the +// library never becomes a row, and neither does a file whose folder resolves +// outside every root. +func TestEligibleUnmatched_SymlinkOutsideRoot(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + secret := filepath.Join(outside, "secret.epub") + if err := os.WriteFile(secret, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "Author"), 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "Author", "Linked.epub") + if err := os.Symlink(secret, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + linkedDir := filepath.Join(root, "Elsewhere") + if err := os.Symlink(outside, linkedDir); err != nil { + t.Fatal(err) + } + real := filepath.Join(root, "Author", "Real.epub") + if err := os.WriteFile(real, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + linkInfo, err := os.Lstat(link) + if err != nil { + t.Fatal(err) + } + + in := []unmatchedScanFile{ + {path: link, format: models.MediaTypeEbook, mode: linkInfo.Mode()}, + {path: filepath.Join(linkedDir, "secret.epub"), format: models.MediaTypeEbook, mode: 0o644}, + {path: real, format: models.MediaTypeEbook, mode: 0o644}, + } + got := eligibleUnmatched(in, []string{root}) + if len(got) != 1 || got[0].path != real { + t.Fatalf("eligible = %+v, want only the real file", got) + } +} + +// TestScanLibrary_SymlinkedBookIsNotListed runs S10 through a real scan. +func TestScanLibrary_SymlinkedBookIsNotListed(t *testing.T) { + s, _, _, _, _, libraryDir, ctx := unmatchedFixture(t) + outside := t.TempDir() + secret := filepath.Join(outside, "Stolen.epub") + writeEpubAt(t, secret, "", "", "") + if err := os.MkdirAll(filepath.Join(libraryDir, "Someone"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(secret, filepath.Join(libraryDir, "Someone", "Stolen.epub")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + writeEpubAt(t, filepath.Join(libraryDir, "Someone", "Honest.epub"), "", "", "") + + s.ScanLibrary(ctx) + + units := readUnmatchedFiles(t, ctx, s) + if len(units) != 1 || filepath.Base(units[0].UnitPath) != "Honest.epub" { + t.Fatalf("units = %+v, want only Honest.epub", units) + } +} + +// TestScanLibrary_RanksCandidatesFromTheCatalogue: a unit close to a book of +// its author, but below the reconcile's 0.85, is stored with that book as a +// suggestion. Only in memory catalogue rows are used. +func TestScanLibrary_RanksCandidatesFromTheCatalogue(t *testing.T) { + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) + author := &models.Author{ForeignID: "ol:weir", Name: "Andy Weir", SortName: "Weir, Andy", MetadataProvider: "openlibrary"} + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + var martian int64 + for _, title := range []string{"The Martian", "Artemis", "Project Hail Mary"} { + b := &models.Book{ForeignID: "ol:" + title, AuthorID: author.ID, Title: title, Status: models.BookStatusWanted, + MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := books.Create(ctx, b); err != nil { + t.Fatal(err) + } + if title == "The Martian" { + martian = b.ID + } + } + path := filepath.Join(libraryDir, "Andy Weir", "A Martyrs Tale", "A Martyrs Tale.epub") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + writeEpubAt(t, path, "", "", "") + + s.ScanLibrary(ctx) + + units := readUnmatchedFiles(t, ctx, s) + if len(units) != 1 { + t.Fatalf("units = %+v", units) + } + c := units[0].Candidates + if len(c) == 0 || c[0].BookID != martian || c[0].Score < candidateThreshold || c[0].Score >= 0.85 { + t.Fatalf("candidates = %+v, want The Martian first with a score in [0.60, 0.85)", c) + } + if units[0].AuthorFolder != "Andy Weir" || units[0].TopScore != c[0].Score { + t.Errorf("unit = %+v", units[0]) + } +} + +// countingTransport fails the test on any outbound request. +type countingTransport struct{ calls int } + +func (c *countingTransport) RoundTrip(*http.Request) (*http.Response, error) { + c.calls++ + return nil, fmt.Errorf("unexpected outbound request during a library scan") +} + +// TestScanLibrary_MakesNoOutboundCalls is T5 for the scan: grouping and +// candidate ranking never reach a metadata provider. +func TestScanLibrary_MakesNoOutboundCalls(t *testing.T) { + spy := &countingTransport{} + prev := http.DefaultTransport + http.DefaultTransport = spy + t.Cleanup(func() { http.DefaultTransport = prev }) + + s, _, books, authors, _, libraryDir, ctx := unmatchedFixture(t) + author := &models.Author{ForeignID: "ol:x", Name: "Ann Leckie", SortName: "Leckie, Ann", MetadataProvider: "openlibrary"} + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + if err := books.Create(ctx, &models.Book{ForeignID: "ol:aj", AuthorID: author.ID, Title: "Ancillary Justice", + Status: models.BookStatusWanted, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"}); err != nil { + t.Fatal(err) + } + for _, rel := range []string{"Ann Leckie/Provenance.epub", "Nobody/Unknown.epub"} { + p := filepath.Join(libraryDir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + writeEpubAt(t, p, "", "", "") + } + + s.ScanLibrary(ctx) + + if spy.calls != 0 { + t.Fatalf("outbound calls during scan = %d, want 0", spy.calls) + } + if n := len(readUnmatchedFiles(t, ctx, s)); n != 2 { + t.Fatalf("units = %d, want 2", n) + } +} + +// TestAudioTrackedByFolder_DiscSet: adoption registers a disc set by its book +// folder, and the next scan must count every disc's tracks as tracked. +func TestAudioTrackedByFolder_DiscSet(t *testing.T) { + tracked := map[string]bool{filepath.Clean("/lib/A/Book"): true} + for _, p := range []string{"/lib/A/Book/01.mp3", "/lib/A/Book/CD1/01.mp3", "/lib/A/Book/Disc 2/05.mp3"} { + if !audioTrackedByFolder(tracked, filepath.Clean(p)) { + t.Errorf("%s not tracked", p) + } + } + for _, p := range []string{"/lib/A/Other/01.mp3", "/lib/A/Book/Extras/01.mp3"} { + if audioTrackedByFolder(tracked, filepath.Clean(p)) { + t.Errorf("%s tracked, want untracked", p) + } + } +} + +// BenchmarkGroupUnmatched is T6: grouping at the 50,000 file cap, a library +// of audiobook folders and loose ebooks. +func BenchmarkGroupUnmatched(b *testing.B) { + files := make([]unmatchedScanFile, 0, maxUnmatchedFiles) + for i := 0; len(files) < maxUnmatchedFiles; i++ { + author := fmt.Sprintf("/lib/Author %03d", i%500) + if i%2 == 0 { + for track := range 20 { + files = append(files, audioFile(fmt.Sprintf("%s/Book %05d/%02d.mp3", author, i, track), fmt.Sprintf("Book %05d", i))) + } + } else { + files = append(files, ebookFile(fmt.Sprintf("%s/Book %05d.epub", author, i))) + } + } + files = files[:maxUnmatchedFiles] + roots := []string{"/lib"} + b.ResetTimer() + for b.Loop() { + groups, _ := groupUnmatched(files, roots) + if len(groups) == 0 { + b.Fatal("no groups") + } + } +} From 11aefa67188f1bcafcd611b7fe241a1b2c242c3a Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 01:55:14 -0300 Subject: [PATCH 05/18] test(api): pin every add core error status and body, refuse search without sync Review found that changing the 422 and 404 add book statuses, or the 502 create author status, left the whole internal/api suite green. The error mapping tests now cover every sentinel and typed error of both cores with the exact status and the exact JSON body, including the 409 bodies that carry the existing book or canonical author. Documents what SkipCatalogueSync skips on each core. On the add book core it removes only the single work fallback; SearchOnAdd still runs. On the create author core it skips the whole sync, and the search on add only ever runs inside that sync, so SearchOnAdd with SkipCatalogueSync would accept a search and never perform it. createAuthorCore now refuses that pair with errCreateAuthorSearchNeedsSync. No HTTP request can set SkipCatalogueSync, so the handler's behaviour is unchanged. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- internal/api/add_book_core.go | 7 ++ internal/api/add_book_core_test.go | 157 ++++++++++++++++++++++++----- internal/api/create_author_core.go | 19 ++++ 3 files changed, 158 insertions(+), 25 deletions(-) diff --git a/internal/api/add_book_core.go b/internal/api/add_book_core.go index ca853d85a..f942da75d 100644 --- a/internal/api/add_book_core.go +++ b/internal/api/add_book_core.go @@ -34,6 +34,13 @@ type addBookParams struct { // when the direct insert produced no row. With it set, the core looks // for the row once instead of polling, since nothing it started could // still create it. False keeps the handler's behaviour. + // + // That fallback is the only sync this core ever starts, so the skip + // removes exactly one author works call and the row it would create. + // It skips nothing else: the fallback is a single work run, which never + // refreshes the author profile (description, image), never relinks a + // Calibre author and never searches. SearchOnAdd is independent of it + // and still searches for the added book, so the two may be combined. SkipCatalogueSync bool } diff --git a/internal/api/add_book_core_test.go b/internal/api/add_book_core_test.go index 64d25bbb6..1f44fb425 100644 --- a/internal/api/add_book_core_test.go +++ b/internal/api/add_book_core_test.go @@ -328,33 +328,140 @@ func TestCreateAuthorCore_SkipCatalogueSyncFiresNoFetch(t *testing.T) { } } -// The handler maps every sentinel to the status and body it answered with -// before the split. This pins the table against the literal bodies. +// errorResponseCase is one row of the error mapping tables below. body holds +// the exact JSON object the handler must write, keys and values both. +type errorResponseCase struct { + name string + err error + status int + body map[string]any +} + +func assertErrorResponses(t *testing.T, write func(http.ResponseWriter, *http.Request, error), cases []errorResponseCase) { + t.Helper() + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := httptest.NewRecorder() + write(rec, httptest.NewRequest(http.MethodPost, "/api/v1/test", nil), tc.err) + if rec.Code != tc.status { + t.Errorf("status = %d, want %d", rec.Code, tc.status) + } + var got any + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("body %q: %v", rec.Body.String(), err) + } + // Round trip the expectation through JSON so numbers and nested + // structs compare in the same shape the handler's body decodes to. + wantRaw, err := json.Marshal(tc.body) + if err != nil { + t.Fatal(err) + } + var want any + if err := json.Unmarshal(wantRaw, &want); err != nil { + t.Fatal(err) + } + gotJSON, _ := json.Marshal(got) + wantJSON, _ := json.Marshal(want) + if string(gotJSON) != string(wantJSON) { + t.Errorf("body = %s\nwant %s", gotJSON, wantJSON) + } + }) + } +} + +func errorBody(msg string) map[string]any { return map[string]any{"error": msg} } + +// Every error addBookCore can return, with the literal status and body the +// AddBook handler answered with before the split. The bodies are spelled out +// here rather than read from addBookErrorResponses, so a change to that table +// fails this test. func TestWriteAddBookError_Statuses(t *testing.T) { h := &AuthorHandler{} - cases := []struct { - err error - status int - body string - }{ - {errAddBookForeignBookIDRequired, http.StatusBadRequest, "foreignBookId required"}, - {errAddBookInvalidMediaType, http.StatusBadRequest, "mediaType must be 'ebook', 'audiobook', or 'both'"}, - {errAddBookAuthorExists, http.StatusConflict, "author already exists"}, - {errAddBookAuthorUnresolved, http.StatusInternalServerError, "could not resolve author"}, - {errAddBookCancelled, http.StatusGatewayTimeout, "request cancelled"}, - {errAddBookHeldByAnotherUser, http.StatusConflict, "book is held by another user"}, - {&addBookLookupError{Err: errors.New("look up book metadata: boom")}, http.StatusBadGateway, "look up book metadata: boom"}, - {errors.New("disk on fire"), http.StatusInternalServerError, "internal server error"}, + existing := &models.Book{ID: 42, ForeignID: "OL1W", Title: "Kept"} + assertErrorResponses(t, h.writeAddBookError, []errorResponseCase{ + {"foreign book id required", errAddBookForeignBookIDRequired, http.StatusBadRequest, + errorBody("foreignBookId required")}, + {"invalid media type", errAddBookInvalidMediaType, http.StatusBadRequest, + errorBody("mediaType must be 'ebook', 'audiobook', or 'both'")}, + {"author metadata unavailable", errAddBookAuthorMetadataUnavailable, http.StatusUnprocessableEntity, + errorBody("Author metadata unavailable for this result. Add the author manually first (Authors → Add Author by name), then try again.")}, + {"author exists", errAddBookAuthorExists, http.StatusConflict, + errorBody("author already exists")}, + {"author unresolved", errAddBookAuthorUnresolved, http.StatusInternalServerError, + errorBody("could not resolve author")}, + {"cancelled", errAddBookCancelled, http.StatusGatewayTimeout, + errorBody("request cancelled")}, + {"not found after sync", errAddBookNotFound, http.StatusNotFound, + errorBody("book not found after author sync — try again shortly")}, + {"held by another user", errAddBookHeldByAnotherUser, http.StatusConflict, + errorBody("book is held by another user")}, + {"book in library", &bookInLibraryError{Book: existing}, http.StatusConflict, map[string]any{ + "error": "book already in your library; change its format or monitoring from the book page", + "existingBookId": existing.ID, + "existingBook": existing, + }}, + {"primary provider unavailable", &primaryProviderUnavailableError{Primary: "hardcover"}, http.StatusServiceUnavailable, + errorBody("The primary metadata provider (hardcover) did not answer, so no record from another provider was used. Please try again once it responds.")}, + {"book lookup failed", &addBookLookupError{Err: errors.New("look up book metadata: boom")}, http.StatusBadGateway, + errorBody("look up book metadata: boom")}, + {"unknown error", errors.New("disk on fire"), http.StatusInternalServerError, + errorBody("internal server error")}, + }) + // A sentinel added to the table without a row above would go unchecked. + if len(addBookErrorResponses) != 8 { + t.Errorf("addBookErrorResponses has %d rows, this test covers 8; add a case for the new one", len(addBookErrorResponses)) } - for _, tc := range cases { - rec := httptest.NewRecorder() - h.writeAddBookError(rec, httptest.NewRequest(http.MethodPost, "/api/v1/author/book", nil), tc.err) - var got map[string]any - if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { - t.Fatalf("%v: body %q: %v", tc.err, rec.Body.String(), err) - } - if rec.Code != tc.status || got["error"] != tc.body { - t.Errorf("%v: got %d %v, want %d %q", tc.err, rec.Code, got["error"], tc.status, tc.body) - } +} + +// Every error createAuthorCore can return, with the literal status and body +// the Create handler answered with before the split. +func TestWriteCreateAuthorError_Statuses(t *testing.T) { + h := &AuthorHandler{} + canonical := &models.Author{ID: 7, ForeignID: "OL7A", Name: "Canon"} + assertErrorResponses(t, h.writeCreateAuthorError, []errorResponseCase{ + {"fields required", errCreateAuthorFieldsRequired, http.StatusBadRequest, + errorBody("foreignAuthorId and authorName required")}, + {"invalid monitor new items", errCreateAuthorInvalidMonitorNewItems, http.StatusBadRequest, + errorBody("invalid monitorNewItems")}, + {"search needs sync", errCreateAuthorSearchNeedsSync, http.StatusBadRequest, + errorBody("searchOnAdd requires the catalogue sync")}, + {"invalid monitor option", &createAuthorOptionError{Err: errors.New("monitorMode must be one of: all, future, latest, none")}, http.StatusBadRequest, + errorBody("monitorMode must be one of: all, future, latest, none")}, + {"upstream lookup failed", &createAuthorLookupError{Err: errors.New("openlibrary: upstream 502")}, http.StatusBadGateway, + errorBody("openlibrary: upstream 502")}, + {"conflict with canonical author", &authorConflictError{Canonical: canonical, Message: "author already exists"}, http.StatusConflict, map[string]any{ + "error": "author already exists", + "canonicalAuthorId": canonical.ID, + "canonicalAuthor": canonical, + }}, + {"conflict without canonical author", &authorConflictError{Message: "author name resolves ambiguously — merge manually"}, http.StatusConflict, + errorBody("author name resolves ambiguously — merge manually")}, + {"unknown error", errors.New("disk on fire"), http.StatusInternalServerError, + errorBody("internal server error")}, + }) + if len(createAuthorErrorResponses) != 3 { + t.Errorf("createAuthorErrorResponses has %d rows, this test covers 3; add a case for the new one", len(createAuthorErrorResponses)) + } +} + +// SearchOnAdd only ever runs inside the catalogue sync, so asking for it with +// the sync skipped is refused before anything is written. +func TestCreateAuthorCore_SearchOnAddWithSkipIsRefused(t *testing.T) { + provider := wellsCreateProvider() + f := newCoreFixture(t, provider) + p := wellsCreateParams + p.SkipCatalogueSync = true + p.SearchOnAdd = true + + _, err := f.h.createAuthorCore(context.Background(), p) + if !errors.Is(err, errCreateAuthorSearchNeedsSync) { + t.Fatalf("err = %v, want errCreateAuthorSearchNeedsSync", err) + } + f.drain(t) + if got, _ := f.authors.GetByForeignID(context.Background(), "OL39307A"); got != nil { + t.Errorf("refused call still created author %+v", got) + } + if n := provider.worksCalls.Load(); n != 0 { + t.Errorf("author works calls = %d, want 0", n) } } diff --git a/internal/api/create_author_core.go b/internal/api/create_author_core.go index e5dd82468..285a35664 100644 --- a/internal/api/create_author_core.go +++ b/internal/api/create_author_core.go @@ -36,6 +36,16 @@ type createAuthorParams struct { MediaType string // SkipCatalogueSync creates or relinks the author without starting its // catalogue sync. False keeps the handler's behaviour. + // + // The sync is everything that happens to the author after the row is + // written, so skipping it skips all of: the author profile refresh + // (description, image, disambiguation, ratings), the Calibre relink, the + // secondary provider identities, the catalogue works fetch and its + // Audible supplement, every book row the sync would create along with + // the library file check for each, the sync summary shown on the author + // page, and the search on add, which only ever runs inside that sync. + // Because SearchOnAdd would silently do nothing, the core refuses the + // combination with errCreateAuthorSearchNeedsSync. SkipCatalogueSync bool } @@ -54,6 +64,11 @@ type createAuthorResult struct { var ( errCreateAuthorFieldsRequired = errors.New("create author: foreignAuthorId and authorName required") errCreateAuthorInvalidMonitorNewItems = errors.New("create author: invalid monitorNewItems") + // errCreateAuthorSearchNeedsSync refuses SearchOnAdd with + // SkipCatalogueSync. The search runs inside the catalogue sync, so the + // pair would accept a search request and never search. Only a non HTTP + // caller can set SkipCatalogueSync, so the handler never returns it. + errCreateAuthorSearchNeedsSync = errors.New("create author: searchOnAdd requires the catalogue sync") ) var createAuthorErrorResponses = []struct { @@ -62,6 +77,7 @@ var createAuthorErrorResponses = []struct { }{ {errCreateAuthorFieldsRequired, "foreignAuthorId and authorName required"}, {errCreateAuthorInvalidMonitorNewItems, "invalid monitorNewItems"}, + {errCreateAuthorSearchNeedsSync, "searchOnAdd requires the catalogue sync"}, } // createAuthorOptionError is an invalid monitorMode or monitorLatestCount. @@ -126,6 +142,9 @@ func (h *AuthorHandler) createAuthorCore(ctx context.Context, req createAuthorPa if req.ForeignID == "" || req.Name == "" { return createAuthorResult{}, errCreateAuthorFieldsRequired } + if req.SearchOnAdd && req.SkipCatalogueSync { + return createAuthorResult{}, errCreateAuthorSearchNeedsSync + } monitorMode, monitorLatestCount, err := h.resolveCreateMonitorOptions(ctx, req.MonitorMode, req.MonitorLatestCount) monitorNewItems := models.DefaultAuthorMonitorNewItems if req.MonitorNewItems != nil { From dceb018bf14f7e339604b3364792f18d4412e02d Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 02:01:32 -0300 Subject: [PATCH 06/18] feat(api): adopt, undo and ignore unmatched library books Adds the admin only /library/unmatched routes. The list filters, sorts through a fixed switch, returns facets only when asked, and hydrates every book a page points at in one query. Adopt registers a unit's files in place against a book already in the library, or against a metadata result added through addBookCore unmonitored, in the adopted format, with no search and SkipCatalogueSync set. No route accepts a path. A request claims its row with a compare and swap, re-checks every file with Lstat and LibraryRoots.ResolveContained, refuses with 409 when a file already belongs to a book, and compensates on failure. Undo untracks exactly what the adoption registered and removes a book or author only when the row recorded creating it and nothing else holds it. The scanner is wired to store unmatched units, and ScanRunning exposes the in flight flag for the page's scan status. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- cmd/bindery/adoption_routes_test.go | 112 ++++++ cmd/bindery/main.go | 8 + cmd/bindery/sensitive_routes.go | 29 ++ internal/api/adoption.go | 435 ++++++++++++++++++++++ internal/api/adoption_adopt.go | 365 +++++++++++++++++++ internal/api/adoption_test.go | 515 +++++++++++++++++++++++++++ internal/importer/unmatched_units.go | 6 + 7 files changed, 1470 insertions(+) create mode 100644 cmd/bindery/adoption_routes_test.go create mode 100644 internal/api/adoption.go create mode 100644 internal/api/adoption_adopt.go create mode 100644 internal/api/adoption_test.go diff --git a/cmd/bindery/adoption_routes_test.go b/cmd/bindery/adoption_routes_test.go new file mode 100644 index 000000000..e62e018bb --- /dev/null +++ b/cmd/bindery/adoption_routes_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "sort" + "strings" + "testing" + + "github.com/go-chi/chi/v5" + + "github.com/vavallee/bindery/internal/auth" +) + +// stubAdoptionHandler records which adoption route ran. +type stubAdoptionHandler struct{ called []string } + +func (h *stubAdoptionHandler) record(name string, w http.ResponseWriter) { + h.called = append(h.called, name) + w.WriteHeader(http.StatusNoContent) +} + +func (h *stubAdoptionHandler) List(w http.ResponseWriter, _ *http.Request) { h.record("list", w) } +func (h *stubAdoptionHandler) Summary(w http.ResponseWriter, _ *http.Request) { h.record("summary", w) } +func (h *stubAdoptionHandler) Adopt(w http.ResponseWriter, _ *http.Request) { h.record("adopt", w) } +func (h *stubAdoptionHandler) Undo(w http.ResponseWriter, _ *http.Request) { h.record("undo", w) } +func (h *stubAdoptionHandler) Ignore(w http.ResponseWriter, _ *http.Request) { h.record("ignore", w) } +func (h *stubAdoptionHandler) Unignore(w http.ResponseWriter, _ *http.Request) { + h.record("unignore", w) +} +func (h *stubAdoptionHandler) IgnoreMany(w http.ResponseWriter, _ *http.Request) { + h.record("ignore-many", w) +} + +// adoptionRoutes is every route registerAdoptionRoutes mounts, with a concrete +// path for each and the handler it must reach. +var adoptionRoutes = []struct { + method, pattern, path, called string +}{ + {http.MethodGet, "/library/unmatched", "/library/unmatched", "list"}, + {http.MethodGet, "/library/unmatched/summary", "/library/unmatched/summary", "summary"}, + {http.MethodPost, "/library/unmatched/ignore", "/library/unmatched/ignore", "ignore-many"}, + {http.MethodPost, "/library/unmatched/{id}/adopt", "/library/unmatched/7/adopt", "adopt"}, + {http.MethodPost, "/library/unmatched/{id}/undo", "/library/unmatched/7/undo", "undo"}, + {http.MethodPost, "/library/unmatched/{id}/ignore", "/library/unmatched/7/ignore", "ignore"}, + {http.MethodPost, "/library/unmatched/{id}/unignore", "/library/unmatched/7/unignore", "unignore"}, +} + +// TestAdoptionRoutesEnumerated walks the mounted router so a route added to +// registerAdoptionRoutes without a row in adoptionRoutes (and so without the +// admin checks below) fails here. +func TestAdoptionRoutesEnumerated(t *testing.T) { + router := chi.NewRouter() + registerAdoptionRoutes(router, &stubAdoptionHandler{}) + var mounted, want []string + if err := chi.Walk(router, func(method, route string, _ http.Handler, _ ...func(http.Handler) http.Handler) error { + mounted = append(mounted, method+" "+route) + return nil + }); err != nil { + t.Fatal(err) + } + for _, r := range adoptionRoutes { + want = append(want, r.method+" "+r.pattern) + } + sort.Strings(mounted) + sort.Strings(want) + if strings.Join(mounted, "\n") != strings.Join(want, "\n") { + t.Fatalf("mounted adoption routes:\n%s\nwant:\n%s", strings.Join(mounted, "\n"), strings.Join(want, "\n")) + } +} + +// TestAdoptionRoutesRequireAdmin: the rows carry absolute library paths and +// adopt writes authors, books and book files, so a user role, or no role at +// all, is refused before any handler runs. +func TestAdoptionRoutesRequireAdmin(t *testing.T) { + for _, role := range []string{"user", ""} { + for _, rt := range adoptionRoutes { + t.Run(role+" "+rt.method+" "+rt.path, func(t *testing.T) { + h := &stubAdoptionHandler{} + router := chi.NewRouter() + registerAdoptionRoutes(router, h) + req := httptest.NewRequest(rt.method, rt.path, nil) + if role != "" { + req = req.WithContext(auth.WithUserRole(req.Context(), role)) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden || len(h.called) != 0 { + t.Fatalf("status = %d, called %v; want 403 and no handler", rec.Code, h.called) + } + }) + } + } +} + +// TestAdoptionRoutesAllowAdmin is the symmetry case. +func TestAdoptionRoutesAllowAdmin(t *testing.T) { + for _, rt := range adoptionRoutes { + t.Run(rt.method+" "+rt.path, func(t *testing.T) { + h := &stubAdoptionHandler{} + router := chi.NewRouter() + registerAdoptionRoutes(router, h) + req := httptest.NewRequest(rt.method, rt.path, nil) + req = req.WithContext(auth.WithUserRole(req.Context(), "admin")) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusNoContent || len(h.called) != 1 || h.called[0] != rt.called { + t.Fatalf("status = %d, called %v; want 204 and %s", rec.Code, h.called, rt.called) + } + }) + } +} diff --git a/cmd/bindery/main.go b/cmd/bindery/main.go index f5a2848aa..284eba433 100644 --- a/cmd/bindery/main.go +++ b/cmd/bindery/main.go @@ -701,6 +701,12 @@ func main() { } libraryHandler := api.NewLibraryHandler(importScanner).WithSettings(settingsRepo) + // Library adoption: the scan stores the books it could not match, and the + // Import page's "In your library" view acts on them. Library roots, not + // importRoots: adoption only registers files already in the library. + unmatchedUnitRepo := db.NewUnmatchedUnitRepo(database) + importScanner.WithUnmatchedUnits(unmatchedUnitRepo) + adoptionHandler := api.NewAdoptionHandler(unmatchedUnitRepo, bookRepo, authorRepo, authorHandler, libraryRoots, importScanner, settingsRepo) fileHandler := api.NewFileHandler(bookRepo, cfg.LibraryDir, cfg.AudiobookDir). WithRootFolders(rootFolderRepo) historyHandler := api.NewHistoryHandler(historyRepo, blocklistRepo, bookRepo) @@ -1138,6 +1144,8 @@ func main() { // admin only (#2361); see registerLibraryScanStatusRoute. r.Post("/library/scan", libraryHandler.Scan) registerLibraryScanStatusRoute(r, libraryHandler) + // Library adoption, admin only (see registerAdoptionRoutes). + registerAdoptionRoutes(r, adoptionHandler) // Refresh metadata for ALL authors (background job, #863). Per-selection // bulk refresh lives at /author/bulk; this is the "populate everything" diff --git a/cmd/bindery/sensitive_routes.go b/cmd/bindery/sensitive_routes.go index e0e707456..e2794e887 100644 --- a/cmd/bindery/sensitive_routes.go +++ b/cmd/bindery/sensitive_routes.go @@ -201,6 +201,35 @@ func registerStorageRoutes(r chi.Router, h storageRouteHandler) { }) } +// adoptionRouteHandler is the surface registerAdoptionRoutes needs. +type adoptionRouteHandler interface { + List(http.ResponseWriter, *http.Request) + Summary(http.ResponseWriter, *http.Request) + Adopt(http.ResponseWriter, *http.Request) + Undo(http.ResponseWriter, *http.Request) + Ignore(http.ResponseWriter, *http.Request) + Unignore(http.ResponseWriter, *http.Request) + IgnoreMany(http.ResponseWriter, *http.Request) +} + +// registerAdoptionRoutes mounts /library/unmatched, library adoption, admin +// only. The rows carry absolute library paths (the same disclosure +// registerLibraryScanStatusRoute gates), adopt creates authors and books and +// writes book_files, and the summary behind the nav badge counts them. No +// route takes a path: every row is addressed by id. +func registerAdoptionRoutes(r chi.Router, h adoptionRouteHandler) { + r.Group(func(r chi.Router) { + r.Use(auth.RequireAdmin) + r.Get("/library/unmatched", h.List) + r.Get("/library/unmatched/summary", h.Summary) + r.Post("/library/unmatched/ignore", h.IgnoreMany) + r.Post("/library/unmatched/{id}/adopt", h.Adopt) + r.Post("/library/unmatched/{id}/undo", h.Undo) + r.Post("/library/unmatched/{id}/ignore", h.Ignore) + r.Post("/library/unmatched/{id}/unignore", h.Unignore) + }) +} + // libraryScanStatusRouteHandler is the surface registerLibraryScanStatusRoute // needs. type libraryScanStatusRouteHandler interface { diff --git a/internal/api/adoption.go b/internal/api/adoption.go new file mode 100644 index 000000000..e1f26e0ba --- /dev/null +++ b/internal/api/adoption.go @@ -0,0 +1,435 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/importer" +) + +// Library adoption (/library/unmatched): the books a library scan could not +// match, and the decisions an admin makes about them. Every route is admin +// only (registerAdoptionRoutes). No request carries a path: rows are addressed +// by id and every path is resolved on the server (S9, S10). + +// maxAdoptionMembersShown bounds the file names a row carries for display. +const maxAdoptionMembersShown = 20 + +// adoptionScanner is the part of the scanner the adoption list reads. +type adoptionScanner interface { + ScanRunning() bool +} + +// adoptionBookAdder is the add book core (see add_book_core.go). +type adoptionBookAdder interface { + addBookCore(ctx context.Context, req addBookParams) (addBookResult, error) +} + +// adoptionUnitStore is the unmatched unit repository surface the handler uses +// (db.UnmatchedUnitRepo). An interface so a test can count the queries a page +// costs. +type adoptionUnitStore interface { + List(ctx context.Context, q db.UnmatchedListQuery) ([]db.UnmatchedUnit, int, error) + Facets(ctx context.Context, q db.UnmatchedListQuery) (db.UnmatchedFacets, error) + Summary(ctx context.Context) (db.UnmatchedSummary, error) + Get(ctx context.Context, id int64) (*db.UnmatchedUnit, error) + BookRefs(ctx context.Context, ids []int64) (map[int64]db.UnmatchedBookRef, error) + ClaimState(ctx context.Context, id int64, from, to string) (bool, error) + CompleteAdoption(ctx context.Context, id int64, rec db.AdoptionRecord) (bool, error) + CompleteUndo(ctx context.Context, id int64) (bool, error) + IgnorePending(ctx context.Context, ids []int64, authorFolder string) (int64, error) + BookReferencedElsewhere(ctx context.Context, bookID, exceptID int64) (bool, error) + AuthorReferencedElsewhere(ctx context.Context, authorID, exceptID int64) (bool, error) +} + +// AdoptionHandler serves the adoption view on the Import page. +type AdoptionHandler struct { + units adoptionUnitStore + books *db.BookRepo + authors *db.AuthorRepo + adder adoptionBookAdder + roots *LibraryRoots + scanner adoptionScanner + settings *db.SettingsRepo + // registerFile records one file against a book and reports whether this + // call inserted it. A seam so tests can fail registration part way. + registerFile func(ctx context.Context, bookID int64, format, path string) (bool, error) +} + +// NewAdoptionHandler wires the adoption routes. roots must be the library +// roots (not the download directories): adoption only ever registers files +// that already live in the library. +func NewAdoptionHandler(units *db.UnmatchedUnitRepo, books *db.BookRepo, authors *db.AuthorRepo, + authorHandler *AuthorHandler, roots *LibraryRoots, scanner *importer.Scanner, settings *db.SettingsRepo) *AdoptionHandler { + h := &AdoptionHandler{units: units, books: books, authors: authors, roots: roots, settings: settings} + if authorHandler != nil { + h.adder = authorHandler + } + if scanner != nil { + h.scanner = scanner + } + h.registerFile = books.AddBookFileIfMissing + return h +} + +// adoptionCandidate is one suggestion with its book. +type adoptionCandidate struct { + Book db.UnmatchedBookRef `json:"book"` + Score float64 `json:"score"` +} + +// adoptionItem is one row as the web sees it. +type adoptionItem struct { + ID int64 `json:"id"` + Kind string `json:"kind"` + Format string `json:"format"` + FileCount int `json:"fileCount"` + SizeBytes int64 `json:"sizeBytes"` + RelPath string `json:"relPath"` + RootPath string `json:"rootPath"` + AuthorFolder string `json:"authorFolder"` + ParsedTitle string `json:"parsedTitle"` + ParsedAuthor string `json:"parsedAuthor"` + Reason string `json:"reason"` + Candidates []adoptionCandidate `json:"candidates"` + TopScore float64 `json:"topScore"` + State string `json:"state"` + Book *db.UnmatchedBookRef `json:"book,omitempty"` + // BookCreated and AuthorCreated say whether the adoption added them, which + // is also what Undo would remove. + BookCreated bool `json:"bookCreated"` + AuthorCreated bool `json:"authorCreated"` + Members []string `json:"members"` + FirstSeenAt time.Time `json:"firstSeenAt"` + ResolvedAt *time.Time `json:"resolvedAt,omitempty"` +} + +// adoptionScanStatus is what the page says about the last and current scan. +type adoptionScanStatus struct { + Ran bool `json:"ran"` + RanAt string `json:"ranAt,omitempty"` + Running bool `json:"running"` + FilesFound int `json:"filesFound"` + Truncated bool `json:"truncated"` + Error string `json:"error,omitempty"` + NoFilesFound bool `json:"noFilesFound"` +} + +type adoptionListResponse struct { + Items []adoptionItem `json:"items"` + Total int `json:"total"` + Facets *db.UnmatchedFacets `json:"facets,omitempty"` + Summary db.UnmatchedSummary `json:"summary"` + Scan adoptionScanStatus `json:"scan"` +} + +type adoptionSummaryResponse struct { + db.UnmatchedSummary + Scan adoptionScanStatus `json:"scan"` +} + +// List handles GET /library/unmatched. Facets are computed only when asked +// for (facets=1), which the web does when the state or search changes, not +// on every page turn (P4). Candidates and adopted books for the whole page +// are hydrated in one query. +func (h *AdoptionHandler) List(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + qv := r.URL.Query() + limit, _ := strconv.Atoi(qv.Get("limit")) + offset, _ := strconv.Atoi(qv.Get("offset")) + q := db.UnmatchedListQuery{ + State: qv.Get("state"), + Reason: qv.Get("reason"), + AuthorFolder: qv.Get("authorFolder"), + Format: qv.Get("format"), + Search: qv.Get("search"), + Sort: qv.Get("sort"), + Dir: qv.Get("dir"), + Limit: limit, + Offset: offset, + } + units, total, err := h.units.List(ctx, q) + if err != nil { + writeServerError(w, r, err) + return + } + items, err := h.hydrate(ctx, units) + if err != nil { + writeServerError(w, r, err) + return + } + resp := adoptionListResponse{Items: items, Total: total, Scan: h.scanStatus(ctx)} + if qv.Get("facets") == "1" { + f, err := h.units.Facets(ctx, q) + if err != nil { + writeServerError(w, r, err) + return + } + resp.Facets = &f + } + if resp.Summary, err = h.units.Summary(ctx); err != nil { + writeServerError(w, r, err) + return + } + writeJSON(w, http.StatusOK, resp) +} + +// Summary handles GET /library/unmatched/summary: the counts behind the nav +// badge and the summary strip, and the scan status the page polls while a +// scan runs (P5). +func (h *AdoptionHandler) Summary(w http.ResponseWriter, r *http.Request) { + sum, err := h.units.Summary(r.Context()) + if err != nil { + writeServerError(w, r, err) + return + } + writeJSON(w, http.StatusOK, adoptionSummaryResponse{UnmatchedSummary: sum, Scan: h.scanStatus(r.Context())}) +} + +// hydrate turns rows into items, loading every book they point at at once. +func (h *AdoptionHandler) hydrate(ctx context.Context, units []db.UnmatchedUnit) ([]adoptionItem, error) { + seen := make(map[int64]bool) + var ids []int64 + want := func(id int64) { + if id > 0 && !seen[id] { + seen[id] = true + ids = append(ids, id) + } + } + for i := range units { + want(units[i].BookID) + for _, c := range units[i].Candidates { + want(c.BookID) + } + } + refs, err := h.units.BookRefs(ctx, ids) + if err != nil { + return nil, err + } + items := make([]adoptionItem, len(units)) + for i := range units { + items[i] = toAdoptionItem(&units[i], refs) + } + return items, nil +} + +func toAdoptionItem(u *db.UnmatchedUnit, refs map[int64]db.UnmatchedBookRef) adoptionItem { + it := adoptionItem{ + ID: u.ID, Kind: u.UnitKind, Format: u.Format, FileCount: u.FileCount, SizeBytes: u.SizeBytes, + RelPath: u.RelPath, RootPath: u.RootPath, AuthorFolder: u.AuthorFolder, + ParsedTitle: u.ParsedTitle, ParsedAuthor: u.ParsedAuthor, Reason: u.Reason, + Candidates: []adoptionCandidate{}, TopScore: u.TopScore, State: u.State, + BookCreated: u.CreatedBookID > 0, AuthorCreated: u.CreatedAuthorID > 0, + Members: memberNames(u), FirstSeenAt: u.FirstSeenAt, ResolvedAt: u.ResolvedAt, + } + for _, c := range u.Candidates { + // A suggested book deleted since the scan simply drops out. + if ref, ok := refs[c.BookID]; ok { + it.Candidates = append(it.Candidates, adoptionCandidate{Book: ref, Score: c.Score}) + } + } + if ref, ok := refs[u.BookID]; ok && u.BookID > 0 { + it.Book = &ref + } + return it +} + +// memberNames lists up to maxAdoptionMembersShown member files, relative to +// the unit folder, or by base name for a file unit. +func memberNames(u *db.UnmatchedUnit) []string { + n := min(len(u.MemberPaths), maxAdoptionMembersShown) + out := make([]string, 0, n) + for _, p := range u.MemberPaths[:n] { + name := filepath.Base(p) + if u.UnitKind == db.UnmatchedKindFolder { + if rel, err := filepath.Rel(u.UnitPath, p); err == nil { + name = filepath.ToSlash(rel) + } + } + out = append(out, name) + } + return out +} + +// scanStatus reads the last scan result blob and the live running flag. +func (h *AdoptionHandler) scanStatus(ctx context.Context) adoptionScanStatus { + st := adoptionScanStatus{} + if h.scanner != nil { + st.Running = h.scanner.ScanRunning() + } + if h.settings == nil { + return st + } + setting, err := h.settings.Get(ctx, SettingLibraryLastScan) + if err != nil || setting == nil || setting.Value == "" { + return st + } + var blob struct { + RanAt string `json:"ran_at"` + FilesFound int `json:"files_found"` + UnitsTruncated bool `json:"units_truncated"` + ScanError string `json:"scan_error"` + NoFilesFound bool `json:"no_files_found"` + } + if json.Unmarshal([]byte(setting.Value), &blob) != nil { + return st + } + st.Ran = true + st.RanAt, st.FilesFound, st.Truncated = blob.RanAt, blob.FilesFound, blob.UnitsTruncated + st.Error, st.NoFilesFound = blob.ScanError, blob.NoFilesFound + return st +} + +// unitIDParam reads {id}, answering 400 itself when it is not a number. +func unitIDParam(w http.ResponseWriter, r *http.Request) (int64, bool) { + id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) + if err != nil || id <= 0 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid id"}) + return 0, false + } + return id, true +} + +// writeUnit answers with one row, freshly read and hydrated. +func (h *AdoptionHandler) writeUnit(w http.ResponseWriter, r *http.Request, id int64) { + u, err := h.units.Get(r.Context(), id) + if err != nil { + writeServerError(w, r, err) + return + } + if u == nil { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "unmatched book not found"}) + return + } + items, err := h.hydrate(r.Context(), []db.UnmatchedUnit{*u}) + if err != nil { + writeServerError(w, r, err) + return + } + writeJSON(w, http.StatusOK, items[0]) +} + +// stateConflict answers a compare and swap that lost, naming the state the +// row is actually in so the web can show something truthful. +func (h *AdoptionHandler) stateConflict(w http.ResponseWriter, r *http.Request, id int64) { + u, err := h.units.Get(r.Context(), id) + if err != nil { + writeServerError(w, r, err) + return + } + if u == nil { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "unmatched book not found"}) + return + } + msg := map[string]string{ + db.UnmatchedStatePending: "This book is still waiting for a decision.", + db.UnmatchedStateAdopting: "Someone is adopting this book right now.", + db.UnmatchedStateAdopted: "This book was already adopted.", + db.UnmatchedStateUndoing: "Someone is undoing this adoption right now.", + db.UnmatchedStateIgnored: "This book is ignored.", + }[u.State] + writeJSON(w, http.StatusConflict, map[string]string{"error": msg, "state": u.State}) +} + +// Ignore handles POST /library/unmatched/{id}/ignore. +func (h *AdoptionHandler) Ignore(w http.ResponseWriter, r *http.Request) { + h.moveState(w, r, db.UnmatchedStatePending, db.UnmatchedStateIgnored) +} + +// Unignore handles POST /library/unmatched/{id}/unignore. +func (h *AdoptionHandler) Unignore(w http.ResponseWriter, r *http.Request) { + h.moveState(w, r, db.UnmatchedStateIgnored, db.UnmatchedStatePending) +} + +func (h *AdoptionHandler) moveState(w http.ResponseWriter, r *http.Request, from, to string) { + id, ok := unitIDParam(w, r) + if !ok { + return + } + moved, err := h.units.ClaimState(r.Context(), id, from, to) + if err != nil { + writeServerError(w, r, err) + return + } + if !moved { + h.stateConflict(w, r, id) + return + } + h.writeUnit(w, r, id) +} + +// maxBulkIgnoreIDs bounds one bulk ignore by id. +const maxBulkIgnoreIDs = 500 + +// IgnoreMany handles POST /library/unmatched/ignore with {ids} or +// {authorFolder}. Only pending rows move. +func (h *AdoptionHandler) IgnoreMany(w http.ResponseWriter, r *http.Request) { + var req struct { + IDs []int64 `json:"ids"` + AuthorFolder string `json:"authorFolder"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&req); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + req.AuthorFolder = strings.TrimSpace(req.AuthorFolder) + if (len(req.IDs) == 0) == (req.AuthorFolder == "") { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "send either ids or authorFolder"}) + return + } + if len(req.IDs) > maxBulkIgnoreIDs { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "too many ids"}) + return + } + n, err := h.units.IgnorePending(r.Context(), req.IDs, req.AuthorFolder) + if err != nil { + writeServerError(w, r, err) + return + } + writeJSON(w, http.StatusOK, map[string]int64{"ignored": n}) +} + +// errAdoptionRefused carries a status and sentence back to the handler. +type errAdoptionRefused struct { + status int + msg string +} + +func (e *errAdoptionRefused) Error() string { return e.msg } + +func refuse(status int, msg string) error { return &errAdoptionRefused{status: status, msg: msg} } + +// writeAdoptionError answers any error an adopt or undo returned. +func (h *AdoptionHandler) writeAdoptionError(w http.ResponseWriter, r *http.Request, err error) { + var refused *errAdoptionRefused + if errors.As(err, &refused) { + writeJSON(w, refused.status, map[string]string{"error": refused.msg}) + return + } + var unavailable *primaryProviderUnavailableError + if errors.As(err, &unavailable) { + writePrimaryProviderUnavailable(w, unavailable.Primary) + return + } + var lookup *addBookLookupError + if errors.As(err, &lookup) { + writeJSON(w, http.StatusBadGateway, map[string]string{"error": lookup.Err.Error()}) + return + } + for _, resp := range addBookErrorResponses { + if errors.Is(err, resp.err) { + writeJSON(w, resp.status, map[string]string{"error": resp.body}) + return + } + } + writeServerError(w, r, err) +} diff --git a/internal/api/adoption_adopt.go b/internal/api/adoption_adopt.go new file mode 100644 index 000000000..f122690a5 --- /dev/null +++ b/internal/api/adoption_adopt.go @@ -0,0 +1,365 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/models" +) + +// adoptRequest names the book a unit is, one of two ways: a book already in +// the library (bookId), or a metadata result to add (foreignBookId with its +// author). There is no path field; the unit's files come from its row. +type adoptRequest struct { + BookID int64 `json:"bookId"` + ForeignBookID string `json:"foreignBookId"` + ForeignAuthorID string `json:"foreignAuthorId"` + AuthorName string `json:"authorName"` + // Format optionally overrides the format the scan detected. + Format string `json:"format"` +} + +// Adopt handles POST /library/unmatched/{id}/adopt. +// +// Adoption is the scanner's own reconcile with a person supplying the match: +// the unit's files are registered in place with the same book_files write the +// scan uses (AddBookFileIfMissing is AddBookFile that also reports whether it +// inserted the row, which is what makes Undo exact). Nothing is moved, renamed +// or queued. +// +// A book added from metadata is created unmonitored and with its media type +// set to the adopted format, so a default of "both" cannot leave the other +// format Wanted and have it grabbed unasked. SkipCatalogueSync is set: the add +// never pulls the author's bibliography anyway (#1816), and skipping the +// single work fallback makes a provider failure fail at once instead of +// polling for 15 seconds, and means no background sync can create the book +// after this request has already given up on it. +// +// Atomicity is by compensation. The row is claimed first (S9); every file is +// checked on disk and against the library roots before anything is written +// (S10); if registering fails part way, what was registered is untracked and +// a book or author this request created is removed again. +func (h *AdoptionHandler) Adopt(w http.ResponseWriter, r *http.Request) { + id, ok := unitIDParam(w, r) + if !ok { + return + } + var req adoptRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 16<<10)).Decode(&req); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + req.ForeignBookID = strings.TrimSpace(req.ForeignBookID) + if (req.BookID > 0) == (req.ForeignBookID != "") { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "send either bookId or foreignBookId"}) + return + } + switch req.Format { + case "", models.MediaTypeEbook, models.MediaTypeAudiobook: + default: + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "format must be 'ebook' or 'audiobook'"}) + return + } + + ctx := r.Context() + claimed, err := h.units.ClaimState(ctx, id, db.UnmatchedStatePending, db.UnmatchedStateAdopting) + if err != nil { + writeServerError(w, r, err) + return + } + if !claimed { + h.stateConflict(w, r, id) + return + } + // The request context may be cancelled once work has started; releasing + // the claim and compensating must still happen. + work := context.WithoutCancel(ctx) + if err := h.adopt(work, id, req); err != nil { + if _, rerr := h.units.ClaimState(work, id, db.UnmatchedStateAdopting, db.UnmatchedStatePending); rerr != nil { + slog.Warn("adoption: could not release claim", "unit", id, "error", rerr) + } + h.writeAdoptionError(w, r, err) + return + } + h.writeUnit(w, r, id) +} + +func (h *AdoptionHandler) adopt(ctx context.Context, id int64, req adoptRequest) error { + unit, err := h.units.Get(ctx, id) + if err != nil { + return err + } + if unit == nil { + return refuse(http.StatusNotFound, "unmatched book not found") + } + format := req.Format + if format == "" { + format = unit.Format + } + paths, err := h.registrationPaths(ctx, unit, format) + if err != nil { + return err + } + // Refuse before any side effect when a file already belongs to a book. + if err := h.checkOwnership(ctx, append(paths, unit.MemberPaths...), 0); err != nil { + return err + } + + var book *models.Book + var created addBookResult + if req.BookID > 0 { + if book, err = h.books.GetByID(ctx, req.BookID); err != nil { + return err + } + if book == nil { + return refuse(http.StatusNotFound, "That book is no longer in your library.") + } + } else { + if h.adder == nil { + return refuse(http.StatusServiceUnavailable, "Adding books is not available.") + } + unmonitored := false + created, err = h.adder.addBookCore(ctx, addBookParams{ + ForeignBookID: req.ForeignBookID, + ForeignAuthorID: strings.TrimSpace(req.ForeignAuthorID), + AuthorName: strings.TrimSpace(req.AuthorName), + SearchOnAdd: false, + MediaType: format, + Monitored: &unmonitored, + SkipCatalogueSync: true, + }) + var inLibrary *bookInLibraryError + switch { + case errors.As(err, &inLibrary): + // Already in the library: adopt into that row, as if picked. + book = inLibrary.Book + case err != nil: + return err + default: + book = created.Book + } + } + + rec := db.AdoptionRecord{BookID: book.ID} + if created.BookCreated { + rec.CreatedBookID = book.ID + } + if created.AuthorCreated && created.Author != nil { + rec.CreatedAuthorID = created.Author.ID + } + + registered, regErr := h.register(ctx, book.ID, format, paths, unit.MemberPaths) + if rec.CreatedBookID > 0 { + // A book this request created may also have picked up a file from + // the add's own library lookup. Everything on it came from this + // request, so Undo, or the compensation below, takes all of it. + if files, err := h.books.ListFiles(ctx, book.ID); err != nil { + if regErr == nil { + regErr = err + } + } else { + registered = make([]string, 0, len(files)) + for _, f := range files { + registered = append(registered, f.Path) + } + } + } + rec.RegisteredPaths = registered + if regErr != nil { + h.reverse(ctx, id, rec) + return regErr + } + done, err := h.units.CompleteAdoption(ctx, id, rec) + if err != nil || !done { + h.reverse(ctx, id, rec) + if err == nil { + err = refuse(http.StatusConflict, "This book changed while it was being adopted. Try again.") + } + return err + } + slog.Info("adoption: registered files in place", "unit", id, "book_id", book.ID, + "files", len(registered), "book_created", rec.CreatedBookID > 0, "author_created", rec.CreatedAuthorID > 0) + return nil +} + +// registrationPaths decides what goes into book_files for a unit and checks +// every file first. An audiobook folder is registered as its folder, the way +// an imported audiobook download is (SetFormatFilePath with the folder); the +// scan counts every track beneath it as tracked. Anything else registers each +// member file. +// +// Each member must still be a regular file (Lstat, so a symlink swapped in +// since the scan is refused) and must resolve inside a library root: the row +// may be hours old (S10). The path registered is the one the scan walked, so +// the next scan recognises it. +func (h *AdoptionHandler) registrationPaths(ctx context.Context, unit *db.UnmatchedUnit, format string) ([]string, error) { + if len(unit.MemberPaths) == 0 { + return nil, refuse(http.StatusUnprocessableEntity, "This book has no files left. Scan again to refresh the list.") + } + for _, p := range unit.MemberPaths { + info, err := os.Lstat(p) + if err != nil { + return nil, refuse(http.StatusUnprocessableEntity, "A file of this book is gone: "+filepath.Base(p)+". Scan again to refresh the list.") + } + if !info.Mode().IsRegular() { + return nil, refuse(http.StatusUnprocessableEntity, filepath.Base(p)+" is not a regular file, so it cannot be adopted.") + } + if _, ok := h.roots.ResolveContained(ctx, p); !ok { + return nil, refuse(http.StatusUnprocessableEntity, filepath.Base(p)+" is outside your library folders, so it cannot be adopted.") + } + } + if unit.UnitKind == db.UnmatchedKindFolder && unit.Format == models.MediaTypeAudiobook && format == models.MediaTypeAudiobook { + if _, ok := h.roots.ResolveContained(ctx, unit.UnitPath); !ok { + return nil, refuse(http.StatusUnprocessableEntity, "This folder is outside your library folders, so it cannot be adopted.") + } + return []string{filepath.Clean(unit.UnitPath)}, nil + } + out := make([]string, len(unit.MemberPaths)) + for i, p := range unit.MemberPaths { + out[i] = filepath.Clean(p) + } + return out, nil +} + +// checkOwnership refuses when any path already belongs to a book other than +// bookID (0 means any book at all). +func (h *AdoptionHandler) checkOwnership(ctx context.Context, paths []string, bookID int64) error { + for _, p := range paths { + owned, err := h.books.PathOwnedByOtherBook(ctx, p, bookID) + if err != nil { + return err + } + if owned { + return refuse(http.StatusConflict, filepath.Base(p)+" already belongs to a book in your library.") + } + } + return nil +} + +// register writes the book_files rows and returns the paths this call +// inserted. On error the caller reverses what it returned. +func (h *AdoptionHandler) register(ctx context.Context, bookID int64, format string, paths, members []string) ([]string, error) { + // Re-check against the resolved book: a concurrent import may have taken + // a file in the moments since the first check. + if err := h.checkOwnership(ctx, append(append([]string{}, paths...), members...), bookID); err != nil { + return nil, err + } + var registered []string + for _, p := range paths { + inserted, err := h.registerFile(ctx, bookID, format, p) + if err != nil { + return registered, err + } + if inserted { + registered = append(registered, p) + } + } + return registered, nil +} + +// reverse undoes a failed adopt's writes: the registered files, then a book +// and an author only if this request created them and nothing else holds +// them. Best effort; every step is logged. +func (h *AdoptionHandler) reverse(ctx context.Context, unitID int64, rec db.AdoptionRecord) { + h.untrack(ctx, rec) + h.removeCreated(ctx, unitID, rec) +} + +// Undo handles POST /library/unmatched/{id}/undo. It untracks exactly the +// paths the adoption registered, then removes a book or author the adoption +// created, but only when the row recorded creating it, it has no other files +// or books, and no other row references it (S9). +func (h *AdoptionHandler) Undo(w http.ResponseWriter, r *http.Request) { + id, ok := unitIDParam(w, r) + if !ok { + return + } + ctx := r.Context() + claimed, err := h.units.ClaimState(ctx, id, db.UnmatchedStateAdopted, db.UnmatchedStateUndoing) + if err != nil { + writeServerError(w, r, err) + return + } + if !claimed { + h.stateConflict(w, r, id) + return + } + work := context.WithoutCancel(ctx) + unit, err := h.units.Get(work, id) + if err != nil || unit == nil { + if _, rerr := h.units.ClaimState(work, id, db.UnmatchedStateUndoing, db.UnmatchedStateAdopted); rerr != nil { + slog.Warn("adoption: could not release undo claim", "unit", id, "error", rerr) + } + if err == nil { + err = refuse(http.StatusNotFound, "unmatched book not found") + } + h.writeAdoptionError(w, r, err) + return + } + rec := db.AdoptionRecord{ + BookID: unit.BookID, CreatedBookID: unit.CreatedBookID, + CreatedAuthorID: unit.CreatedAuthorID, RegisteredPaths: unit.RegisteredPaths, + } + h.untrack(work, rec) + h.removeCreated(work, id, rec) + if _, err := h.units.CompleteUndo(work, id); err != nil { + writeServerError(w, r, err) + return + } + slog.Info("adoption: undone", "unit", id, "book_id", rec.BookID, "files", len(rec.RegisteredPaths)) + h.writeUnit(w, r, id) +} + +// untrack removes the adoption's book_files rows. A path that has since moved +// to a different book (a manual reassign, say) belongs to that book now and is +// left alone. +func (h *AdoptionHandler) untrack(ctx context.Context, rec db.AdoptionRecord) { + for _, p := range rec.RegisteredPaths { + if rec.BookID > 0 { + if other, err := h.books.PathOwnedByOtherBook(ctx, p, rec.BookID); err != nil || other { + continue + } + } + if _, err := h.books.UntrackFilePath(ctx, p); err != nil { + slog.Warn("adoption: could not untrack file", "path", p, "error", err) + } + } +} + +// removeCreated deletes a book and an author the adoption created, each only +// when nothing else still depends on it. +func (h *AdoptionHandler) removeCreated(ctx context.Context, unitID int64, rec db.AdoptionRecord) { + if rec.CreatedBookID > 0 && h.bookIsOnlyOurs(ctx, unitID, rec.CreatedBookID) { + if err := h.books.Delete(ctx, rec.CreatedBookID); err != nil { + slog.Warn("adoption: could not remove created book", "book_id", rec.CreatedBookID, "error", err) + } + } + if rec.CreatedAuthorID > 0 { + books, err := h.books.ListByAuthor(ctx, rec.CreatedAuthorID) + if err != nil || len(books) > 0 { + return + } + if other, err := h.units.AuthorReferencedElsewhere(ctx, rec.CreatedAuthorID, unitID); err != nil || other { + return + } + if err := h.authors.Delete(ctx, rec.CreatedAuthorID); err != nil { + slog.Warn("adoption: could not remove created author", "author_id", rec.CreatedAuthorID, "error", err) + } + } +} + +func (h *AdoptionHandler) bookIsOnlyOurs(ctx context.Context, unitID, bookID int64) bool { + files, err := h.books.ListFiles(ctx, bookID) + if err != nil || len(files) > 0 { + return false + } + other, err := h.units.BookReferencedElsewhere(ctx, bookID, unitID) + return err == nil && !other +} diff --git a/internal/api/adoption_test.go b/internal/api/adoption_test.go new file mode 100644 index 000000000..3de6fdb67 --- /dev/null +++ b/internal/api/adoption_test.go @@ -0,0 +1,515 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/jobs" + "github.com/vavallee/bindery/internal/metadata" + "github.com/vavallee/bindery/internal/models" +) + +type adoptionFixture struct { + h *AdoptionHandler + router chi.Router + units *db.UnmatchedUnitRepo + books *db.BookRepo + authors *db.AuthorRepo + lib string +} + +// newAdoptionFixture wires the handler over an in memory database, a real +// AuthorHandler on the given provider (T1: the stubMetaProvider doubles), and +// a temporary library root. +func newAdoptionFixture(t *testing.T, provider metadata.Provider) adoptionFixture { + t.Helper() + database, err := db.OpenMemory() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + authors := db.NewAuthorRepo(database) + books := db.NewBookRepo(database) + group := jobs.NewGroup(context.Background()) + t.Cleanup(func() { group.Shutdown(5 * time.Second) }) + ah := NewAuthorHandler(authors, nil, books, nil, metadata.NewAggregator(provider), nil, + db.NewMetadataProfileRepo(database), nil).WithJobs(group) + lib := t.TempDir() + if real, err := filepath.EvalSymlinks(lib); err == nil { + lib = real + } + units := db.NewUnmatchedUnitRepo(database) + h := NewAdoptionHandler(units, books, authors, ah, NewLibraryRoots(nil, lib), nil, db.NewSettingsRepo(database)) + r := chi.NewRouter() + r.Get("/library/unmatched", h.List) + r.Get("/library/unmatched/summary", h.Summary) + r.Post("/library/unmatched/ignore", h.IgnoreMany) + r.Post("/library/unmatched/{id}/adopt", h.Adopt) + r.Post("/library/unmatched/{id}/undo", h.Undo) + r.Post("/library/unmatched/{id}/ignore", h.Ignore) + r.Post("/library/unmatched/{id}/unignore", h.Unignore) + return adoptionFixture{h: h, router: r, units: units, books: books, authors: authors, lib: lib} +} + +func (f adoptionFixture) write(t *testing.T, rel string) string { + t.Helper() + p := filepath.Join(f.lib, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("book"), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +// seedUnit stores one unit the way a scan would and returns its id. +func (f adoptionFixture) seedUnit(t *testing.T, u db.UnmatchedUnitScan) int64 { + t.Helper() + if u.UnitKind == "" { + u.UnitKind = db.UnmatchedKindFile + } + if u.Format == "" { + u.Format = models.MediaTypeEbook + } + u.FileCount = len(u.MemberPaths) + u.RootPath = f.lib + if _, err := f.units.ReconcileScan(context.Background(), []db.UnmatchedUnitScan{u}, db.ReconcileScanOptions{SkipDeletion: true}); err != nil { + t.Fatal(err) + } + items, _, err := f.units.List(context.Background(), db.UnmatchedListQuery{Limit: 250}) + if err != nil { + t.Fatal(err) + } + for _, it := range items { + if it.UnitPath == u.UnitPath { + return it.ID + } + } + t.Fatalf("seeded unit %s not listed", u.UnitPath) + return 0 +} + +func (f adoptionFixture) post(t *testing.T, path string, body any) *httptest.ResponseRecorder { + t.Helper() + var buf bytes.Buffer + if body != nil { + if err := json.NewEncoder(&buf).Encode(body); err != nil { + t.Fatal(err) + } + } + rec := httptest.NewRecorder() + f.router.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, path, &buf)) + return rec +} + +func (f adoptionFixture) seedBook(t *testing.T, title string) *models.Book { + t.Helper() + ctx := context.Background() + author := &models.Author{ForeignID: "ol:" + title, Name: "Ann Leckie", SortName: "Leckie, Ann", MetadataProvider: "openlibrary"} + if err := f.authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + b := &models.Book{ForeignID: "ol:b:" + title, AuthorID: author.ID, Title: title, Status: models.BookStatusWanted, + Monitored: true, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary"} + if err := f.books.Create(ctx, b); err != nil { + t.Fatal(err) + } + return b +} + +func decodeItem(t *testing.T, rec *httptest.ResponseRecorder) adoptionItem { + t.Helper() + var it adoptionItem + if err := json.Unmarshal(rec.Body.Bytes(), &it); err != nil { + t.Fatalf("decode %q: %v", rec.Body.String(), err) + } + return it +} + +func filePaths(t *testing.T, books *db.BookRepo, bookID int64) []string { + t.Helper() + files, err := books.ListFiles(context.Background(), bookID) + if err != nil { + t.Fatal(err) + } + out := make([]string, 0, len(files)) + for _, f := range files { + out = append(out, f.Path) + } + return out +} + +// TestAdopt_ExistingBookRegistersInPlaceAndUndoIsExact: adopting into a book +// already in the library registers the file where it is, and undo removes +// exactly that registration and leaves the book. +func TestAdopt_ExistingBookRegistersInPlaceAndUndoIsExact(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + book := f.seedBook(t, "Ancillary Justice") + path := f.write(t, "Ann Leckie/Ancillary Justice.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": book.ID}) + if rec.Code != http.StatusOK { + t.Fatalf("adopt = %d %s", rec.Code, rec.Body.String()) + } + it := decodeItem(t, rec) + if it.State != db.UnmatchedStateAdopted || it.Book == nil || it.Book.ID != book.ID || it.BookCreated || it.AuthorCreated { + t.Fatalf("adopted item = %+v", it) + } + if got := filePaths(t, f.books, book.ID); len(got) != 1 || got[0] != path { + t.Fatalf("book files = %v, want the file in place", got) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("file moved or removed: %v", err) + } + after, _ := f.books.GetByID(context.Background(), book.ID) + if after.Status != models.BookStatusImported || !after.Monitored { + t.Fatalf("book after adopt = status %s monitored %v, want imported and monitoring untouched", after.Status, after.Monitored) + } + + rec = f.post(t, fmt.Sprintf("/library/unmatched/%d/undo", id), nil) + if rec.Code != http.StatusOK || decodeItem(t, rec).State != db.UnmatchedStatePending { + t.Fatalf("undo = %d %s", rec.Code, rec.Body.String()) + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("book files after undo = %v, want none", got) + } + if b, _ := f.books.GetByID(context.Background(), book.ID); b == nil || b.Status != models.BookStatusWanted { + t.Fatalf("existing book after undo = %+v, want kept and wanted again", b) + } +} + +// TestAdopt_CreatesUnmonitoredBookInTheAdoptedFormat: a metadata result is +// added unmonitored with media type set to the unit's format, so nothing is +// left Wanted to be grabbed. An audiobook folder registers as its folder. +// Undo removes the created book and author again. +func TestAdopt_CreatesUnmonitoredBookInTheAdoptedFormat(t *testing.T) { + f := newAdoptionFixture(t, addBookBackCatalogueStub(false)) + var tracks []string + for i := 1; i <= 3; i++ { + tracks = append(tracks, f.write(t, fmt.Sprintf("H. G. Wells/War of the Worlds/%02d.mp3", i))) + } + folder := filepath.Dir(tracks[0]) + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: folder, UnitKind: db.UnmatchedKindFolder, + Format: models.MediaTypeAudiobook, MemberPaths: tracks}) + + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{ + "foreignBookId": "OL27482W", "foreignAuthorId": "OL39307A", "authorName": "H. G. Wells", + }) + if rec.Code != http.StatusOK { + t.Fatalf("adopt = %d %s", rec.Code, rec.Body.String()) + } + it := decodeItem(t, rec) + if !it.BookCreated || !it.AuthorCreated || it.Book == nil { + t.Fatalf("item = %+v, want a created book and author", it) + } + book, _ := f.books.GetByID(context.Background(), it.Book.ID) + if book.Monitored || book.MediaType != models.MediaTypeAudiobook || book.Status != models.BookStatusImported { + t.Fatalf("created book = monitored %v, media %s, status %s; want unmonitored audiobook, imported", + book.Monitored, book.MediaType, book.Status) + } + if got := filePaths(t, f.books, book.ID); len(got) != 1 || got[0] != folder { + t.Fatalf("registered = %v, want the folder %s", got, folder) + } + + authorID := book.AuthorID + rec = f.post(t, fmt.Sprintf("/library/unmatched/%d/undo", id), nil) + if rec.Code != http.StatusOK { + t.Fatalf("undo = %d %s", rec.Code, rec.Body.String()) + } + if b, _ := f.books.GetByID(context.Background(), book.ID); b != nil { + t.Fatalf("created book survived undo: %+v", b) + } + if a, _ := f.authors.GetByID(context.Background(), authorID); a != nil { + t.Fatalf("created author survived undo: %+v", a) + } +} + +// TestAdopt_InjectedFailureStrandsNothing: registration failing part way +// leaves no file row, no created book or author, and the row pending. +func TestAdopt_InjectedFailureStrandsNothing(t *testing.T) { + f := newAdoptionFixture(t, addBookBackCatalogueStub(false)) + epub := f.write(t, "H. G. Wells/War of the Worlds.epub") + mobi := f.write(t, "H. G. Wells/War of the Worlds.mobi") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: epub, MemberPaths: []string{epub, mobi}}) + + real := f.h.registerFile + calls := 0 + f.h.registerFile = func(ctx context.Context, bookID int64, format, path string) (bool, error) { + calls++ + if calls == 2 { + return false, errors.New("disk on fire") + } + return real(ctx, bookID, format, path) + } + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{ + "foreignBookId": "OL27482W", "foreignAuthorId": "OL39307A", "authorName": "H. G. Wells", + }) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("adopt = %d %s, want 500", rec.Code, rec.Body.String()) + } + ctx := context.Background() + if b, _ := f.books.GetByForeignID(ctx, "OL27482W"); b != nil { + t.Fatalf("created book stranded: %+v", b) + } + if a, _ := f.authors.GetByForeignID(ctx, "OL39307A"); a != nil { + t.Fatalf("created author stranded: %+v", a) + } + for _, p := range []string{epub, mobi} { + if owned, _ := f.books.PathOwnedByOtherBook(ctx, p, 0); owned { + t.Fatalf("%s still registered", p) + } + } + if u, _ := f.units.Get(ctx, id); u.State != db.UnmatchedStatePending { + t.Fatalf("unit state = %s, want pending", u.State) + } +} + +// TestAdopt_RefusesAFileAnotherBookOwns is the 409. +func TestAdopt_RefusesAFileAnotherBookOwns(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + owner := f.seedBook(t, "Provenance") + target := f.seedBook(t, "Translation State") + path := f.write(t, "Ann Leckie/Provenance.epub") + if err := f.books.AddBookFile(context.Background(), owner.ID, models.MediaTypeEbook, path); err != nil { + t.Fatal(err) + } + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": target.ID}) + if rec.Code != http.StatusConflict { + t.Fatalf("adopt = %d %s, want 409", rec.Code, rec.Body.String()) + } + if got := filePaths(t, f.books, target.ID); len(got) != 0 { + t.Fatalf("target gained %v", got) + } +} + +// TestAdopt_RejectsSymlinkEscape is S10 at adopt time: the row may be hours +// old, so a member swapped for a symlink, or reached through a symlinked +// folder that leaves the library, is refused and nothing is written. +func TestAdopt_RejectsSymlinkEscape(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + book := f.seedBook(t, "Ancillary Sword") + outside := t.TempDir() + secret := filepath.Join(outside, "secret.epub") + if err := os.WriteFile(secret, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + + swapped := f.write(t, "Ann Leckie/Swapped.epub") + if err := os.Remove(swapped); err != nil { + t.Fatal(err) + } + if err := os.Symlink(secret, swapped); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + escapeDir := filepath.Join(f.lib, "Escape") + if err := os.Symlink(outside, escapeDir); err != nil { + t.Fatal(err) + } + throughDir := filepath.Join(escapeDir, "secret.epub") + + for _, p := range []string{swapped, throughDir} { + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: p, MemberPaths: []string{p}}) + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), map[string]any{"bookId": book.ID}) + if rec.Code != http.StatusUnprocessableEntity { + t.Errorf("adopt %s = %d %s, want 422", p, rec.Code, rec.Body.String()) + } + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("registered through a symlink: %v", got) + } +} + +// TestAdopt_DoubleAdoptAndDoubleUndo is T3: concurrent requests for one row, +// exactly one wins each time. +func TestAdopt_DoubleAdoptAndDoubleUndo(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + book := f.seedBook(t, "Ancillary Mercy") + path := f.write(t, "Ann Leckie/Ancillary Mercy.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + + race := func(action string, body any) (ok, conflict int32) { + var wg sync.WaitGroup + for range 6 { + wg.Add(1) + go func() { + defer wg.Done() + rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/%s", id, action), body) + switch rec.Code { + case http.StatusOK: + atomic.AddInt32(&ok, 1) + case http.StatusConflict: + atomic.AddInt32(&conflict, 1) + default: + t.Errorf("%s = %d %s", action, rec.Code, rec.Body.String()) + } + }() + } + wg.Wait() + return ok, conflict + } + if ok, conflict := race("adopt", map[string]any{"bookId": book.ID}); ok != 1 || conflict != 5 { + t.Fatalf("adopt race: %d ok, %d conflict; want 1 and 5", ok, conflict) + } + if got := filePaths(t, f.books, book.ID); len(got) != 1 { + t.Fatalf("files after adopt race = %v", got) + } + if ok, conflict := race("undo", nil); ok != 1 || conflict != 5 { + t.Fatalf("undo race: %d ok, %d conflict; want 1 and 5", ok, conflict) + } + if got := filePaths(t, f.books, book.ID); len(got) != 0 { + t.Fatalf("files after undo race = %v", got) + } +} + +// TestAdopt_BadRequests: no path field exists, and the body must name one +// target. +func TestAdopt_BadRequests(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + path := f.write(t, "X/y.epub") + id := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: path, MemberPaths: []string{path}}) + for name, body := range map[string]any{ + "neither": map[string]any{}, + "both": map[string]any{"bookId": 1, "foreignBookId": "OL1W"}, + "bad format": map[string]any{"bookId": 1, "format": "both"}, + } { + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/adopt", id), body); rec.Code != http.StatusBadRequest { + t.Errorf("%s: %d, want 400", name, rec.Code) + } + } + if rec := f.post(t, "/library/unmatched/999/adopt", map[string]any{"bookId": 1}); rec.Code != http.StatusConflict && rec.Code != http.StatusNotFound { + t.Errorf("unknown id: %d", rec.Code) + } +} + +// TestIgnore_SingleBulkAndUnignore covers the ignore routes and their state +// guards. +func TestIgnore_SingleBulkAndUnignore(t *testing.T) { + f := newAdoptionFixture(t, &stubMetaProvider{name: "openlibrary"}) + a := f.write(t, "Folder A/one.epub") + b := f.write(t, "Folder A/two.epub") + c := f.write(t, "Folder B/three.epub") + ida := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: a, MemberPaths: []string{a}, AuthorFolder: "Folder A"}) + f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: b, MemberPaths: []string{b}, AuthorFolder: "Folder A"}) + idc := f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: c, MemberPaths: []string{c}, AuthorFolder: "Folder B"}) + + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/ignore", idc), nil); rec.Code != http.StatusOK { + t.Fatalf("ignore = %d", rec.Code) + } + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/ignore", idc), nil); rec.Code != http.StatusConflict { + t.Fatalf("second ignore = %d, want 409", rec.Code) + } + rec := f.post(t, "/library/unmatched/ignore", map[string]any{"authorFolder": "Folder A"}) + var bulk map[string]int64 + _ = json.Unmarshal(rec.Body.Bytes(), &bulk) + if rec.Code != http.StatusOK || bulk["ignored"] != 2 { + t.Fatalf("bulk ignore = %d %s", rec.Code, rec.Body.String()) + } + if rec := f.post(t, "/library/unmatched/ignore", map[string]any{"ids": []int64{ida}, "authorFolder": "x"}); rec.Code != http.StatusBadRequest { + t.Fatalf("bulk with both selectors = %d, want 400", rec.Code) + } + if rec := f.post(t, fmt.Sprintf("/library/unmatched/%d/unignore", ida), nil); rec.Code != http.StatusOK || decodeItem(t, rec).State != db.UnmatchedStatePending { + t.Fatalf("unignore = %d %s", rec.Code, rec.Body.String()) + } +} + +// countingUnitStore counts the book hydration queries a page costs. +type countingUnitStore struct { + *db.UnmatchedUnitRepo + bookRefs atomic.Int32 +} + +func (c *countingUnitStore) BookRefs(ctx context.Context, ids []int64) (map[int64]db.UnmatchedBookRef, error) { + c.bookRefs.Add(1) + return c.UnmatchedUnitRepo.BookRefs(ctx, ids) +} + +// countingProvider counts every metadata provider call. +type countingProvider struct { + metadata.Provider + calls atomic.Int32 +} + +func (p *countingProvider) SearchAuthors(ctx context.Context, q string) ([]models.Author, error) { + p.calls.Add(1) + return p.Provider.SearchAuthors(ctx, q) +} +func (p *countingProvider) SearchBooks(ctx context.Context, q string) ([]models.Book, error) { + p.calls.Add(1) + return p.Provider.SearchBooks(ctx, q) +} +func (p *countingProvider) GetAuthor(ctx context.Context, id string) (*models.Author, error) { + p.calls.Add(1) + return p.Provider.GetAuthor(ctx, id) +} +func (p *countingProvider) GetBook(ctx context.Context, id string) (*models.Book, error) { + p.calls.Add(1) + return p.Provider.GetBook(ctx, id) +} +func (p *countingProvider) GetEditions(ctx context.Context, id string) ([]models.Edition, error) { + p.calls.Add(1) + return p.Provider.GetEditions(ctx, id) +} +func (p *countingProvider) GetBookByISBN(ctx context.Context, isbn string) (*models.Book, error) { + p.calls.Add(1) + return p.Provider.GetBookByISBN(ctx, isbn) +} + +// TestAdoptionList_NoProviderCallsAndOneHydrationQuery is T5: a page with +// several rows, each with suggestions, costs one book query and no provider +// call, with and without facets. +func TestAdoptionList_NoProviderCallsAndOneHydrationQuery(t *testing.T) { + provider := &countingProvider{Provider: &stubMetaProvider{name: "openlibrary"}} + f := newAdoptionFixture(t, provider) + counting := &countingUnitStore{UnmatchedUnitRepo: f.units} + f.h.units = counting + + b1 := f.seedBook(t, "Ancillary Justice") + b2 := f.seedBook(t, "Ancillary Sword") + for i := range 5 { + p := f.write(t, fmt.Sprintf("Ann Leckie/Book %d.epub", i)) + f.seedUnit(t, db.UnmatchedUnitScan{UnitPath: p, MemberPaths: []string{p}, AuthorFolder: "Ann Leckie", ParsedTitle: fmt.Sprintf("Book %d", i), + Candidates: []db.UnmatchedCandidate{{BookID: b1.ID, Score: 0.7}, {BookID: b2.ID, Score: 0.65}}}) + } + + for _, q := range []string{"", "?facets=1", "?facets=1&search=book&sort=title"} { + counting.bookRefs.Store(0) + rec := httptest.NewRecorder() + f.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/library/unmatched"+q, nil)) + if rec.Code != http.StatusOK { + t.Fatalf("list%s = %d %s", q, rec.Code, rec.Body.String()) + } + var resp adoptionListResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Total != 5 || len(resp.Items) != 5 || len(resp.Items[0].Candidates) != 2 || resp.Items[0].Candidates[0].Book.Title != "Ancillary Justice" { + t.Fatalf("list%s = %+v", q, resp) + } + if n := counting.bookRefs.Load(); n != 1 { + t.Errorf("list%s hydration queries = %d, want 1", q, n) + } + if (q != "") != (resp.Facets != nil) { + t.Errorf("list%s facets present = %v", q, resp.Facets != nil) + } + } + if n := provider.calls.Load(); n != 0 { + t.Fatalf("provider calls during list = %d, want 0", n) + } +} diff --git a/internal/importer/unmatched_units.go b/internal/importer/unmatched_units.go index ecdd1e7bc..d6311e806 100644 --- a/internal/importer/unmatched_units.go +++ b/internal/importer/unmatched_units.go @@ -46,6 +46,12 @@ func (s *Scanner) WithUnmatchedUnits(store UnmatchedUnitStore) *Scanner { return s } +// ScanRunning reports whether a library scan is in flight, for the adoption +// page's scan status. +func (s *Scanner) ScanRunning() bool { + return s.scanRunning.Load() +} + // walkedFile is what the library walk already knew about a file (P2): its // size and mode come from the os.FileInfo filepath.Walk hands over, so // grouping costs no extra stat. From 8f330151b2d48887cacce3869e07c286fce98950 Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 02:20:17 -0300 Subject: [PATCH 07/18] fix(scanner): name a disc set after its book folder A multi disc audiobook's tracks parse their title from the disc folder, so the unit read "CD 1". The unit is the book folder above the discs, and it is named after that folder now. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- internal/importer/unmatched_units.go | 5 + internal/importer/unmatched_units_test.go | 9 + web/src/pages/ManualImportPage.tsx | 618 ------------------ .../FolderImportView.test.tsx} | 0 4 files changed, 14 insertions(+), 618 deletions(-) delete mode 100644 web/src/pages/ManualImportPage.tsx rename web/src/pages/{ManualImportPage.test.tsx => import/FolderImportView.test.tsx} (100%) diff --git a/internal/importer/unmatched_units.go b/internal/importer/unmatched_units.go index d6311e806..0aa48712f 100644 --- a/internal/importer/unmatched_units.go +++ b/internal/importer/unmatched_units.go @@ -185,6 +185,11 @@ func groupUnmatched(files []unmatchedScanFile, roots []string) (groups []unmatch } } rep := representative(a.members) + // A disc set's tracks parse their title from the disc folder ("CD 1"); + // the book is the folder above it, so the unit is named after that. + if a.folder != "" && filepath.Dir(a.members[0].path) != a.folder { + rep.title = cleanLayoutTitle(filepath.Base(a.folder)) + } u.ParsedTitle, u.ParsedAuthor, u.Reason = rep.title, rep.author, rep.reason groups = append(groups, unmatchedGroup{unit: u, rep: rep}) } diff --git a/internal/importer/unmatched_units_test.go b/internal/importer/unmatched_units_test.go index dbc6ce568..9150307fb 100644 --- a/internal/importer/unmatched_units_test.go +++ b/internal/importer/unmatched_units_test.go @@ -49,6 +49,15 @@ func TestGroupUnmatched_Table(t *testing.T) { }, want: []want{{"/lib/Weir/Artemis", "folder", "audiobook", "Weir", "Weir/Artemis", "Artemis", 2}}, }, + { + name: "a disc set is named after its book folder, not a disc", + files: []unmatchedScanFile{ + audioFile("/lib/Sanderson/Rhythm of War/CD1/01.mp3", "CD 1"), + audioFile("/lib/Sanderson/Rhythm of War/CD2/01.mp3", "CD 2"), + audioFile("/lib/Sanderson/Rhythm of War/CD2/02.mp3", "CD 2"), + }, + want: []want{{"/lib/Sanderson/Rhythm of War", "folder", "audiobook", "Sanderson", "Sanderson/Rhythm of War", "Rhythm of War", 3}}, + }, { name: "a disc folder straight under the root is its own unit", files: []unmatchedScanFile{ diff --git a/web/src/pages/ManualImportPage.tsx b/web/src/pages/ManualImportPage.tsx deleted file mode 100644 index 023f2b5a6..000000000 --- a/web/src/pages/ManualImportPage.tsx +++ /dev/null @@ -1,618 +0,0 @@ -import { useEffect, useMemo, useState } from 'react' -import { useTranslation } from 'react-i18next' -import { api } from '../api/client' -import { resolveBookQuery } from '../api/booklookup' -import type { BatchImportItem, BatchImportResult, Book, ScanItem } from '../api/client' -import { btn, btnSize } from '../components/buttons' - -// ManualImportPage is the manual-import wizard (#1236). It scans a folder with -// the recursive bulk-import scan (#1434), groups the discovered units by match -// confidence, and lets the user resolve each one before importing: -// - confident: a preselected catalogue match, deselectable and overridable; -// - ambiguous: a candidate picker over the returned catalogue candidates; -// - none: a search box to bind the unit to an EXISTING catalogue book, plus -// a metadata search that creates the book (and its author) when the library -// has no row to bind to (#1719). -// Resolved units are imported (per-unit or in bulk) through the batch endpoint, -// which reports per-item success/failure. - -const MATCH_ORDER: ScanItem['match'][] = ['confident', 'ambiguous', 'none'] - -// RowState is the per-unit resolution the user is building, keyed by unit path. -interface RowState { - // chosen is the catalogue book the unit will import against (a confident - // match, a picked candidate, or a searched existing book). null until resolved. - chosen: Book | null - // format override: '' = auto-detect, or 'ebook' / 'audiobook'. Defaults to the - // scan's detected format. - format: string - // selected marks the unit for the next bulk import. Only meaningful when chosen. - selected: boolean -} - -function bookLabel(b: Book): string { - return b.author?.authorName ? `${b.title} · ${b.author.authorName}` : b.title -} - -function groupHeading(match: ScanItem['match']): string { - switch (match) { - case 'confident': return 'Matched' - case 'ambiguous': return 'Needs a choice' - default: return 'Unmatched' - } -} - -export default function ManualImportPage() { - const { t } = useTranslation() - const [path, setPath] = useState('') - const [scanning, setScanning] = useState(false) - const [items, setItems] = useState(null) - const [truncated, setTruncated] = useState(false) - const [rows, setRows] = useState>({}) - const [results, setResults] = useState>({}) - const [importingPaths, setImportingPaths] = useState>(() => new Set()) - const [scanError, setScanError] = useState('') - const [importError, setImportError] = useState('') - - useEffect(() => { - document.title = 'Manual Import · Bindery' - return () => { document.title = 'Bindery' } - }, []) - - const handleScan = async () => { - const p = path.trim() - if (!p) return - setScanning(true) - setScanError('') - setImportError('') - setItems(null) - setResults({}) - try { - const r = await api.scanFolder(p) - setItems(r.items) - setTruncated(r.truncated) - const init: Record = {} - for (const it of r.items) { - const chosen = it.match === 'confident' && it.book ? it.book : null - init[it.path] = { chosen, format: it.detectedFormat || '', selected: Boolean(chosen) } - } - setRows(init) - } catch (e) { - setScanError(e instanceof Error ? e.message : 'Scan failed') - } finally { - setScanning(false) - } - } - - const patchRow = (unitPath: string, patch: Partial) => - setRows(prev => ({ ...prev, [unitPath]: { ...prev[unitPath], ...patch } })) - - const pickBook = (unitPath: string, book: Book) => - patchRow(unitPath, { chosen: book, selected: true }) - - const toggleSelected = (unitPath: string) => - setRows(prev => { - const r = prev[unitPath] - if (!r || !r.chosen) return prev - return { ...prev, [unitPath]: { ...r, selected: !r.selected } } - }) - - // A path is submittable when it has a chosen book, is selected, and hasn't - // already been accepted by a prior import. A `none` unit the user never - // resolved has chosen=null, so it is never submitted. - const submittablePaths = useMemo(() => { - if (!items) return [] - return items - .map(it => it.path) - .filter(p => { - const r = rows[p] - return Boolean(r?.chosen) && r.selected && !results[p]?.accepted - }) - }, [items, rows, results]) - - const submit = async (paths: string[]) => { - const batch: BatchImportItem[] = [] - for (const p of paths) { - const r = rows[p] - if (!r?.chosen || !r.selected) continue - if (results[p]?.accepted) continue - batch.push({ path: p, bookId: r.chosen.id, format: r.format || undefined }) - } - if (batch.length === 0) return - setImportError('') - setImportingPaths(prev => { const n = new Set(prev); batch.forEach(b => n.add(b.path)); return n }) - try { - const res = await api.batchImport(batch) - setResults(prev => { - const n = { ...prev } - for (const r of res.results) n[r.path] = r - return n - }) - } catch (e) { - setImportError(e instanceof Error ? e.message : 'Import failed') - } finally { - setImportingPaths(prev => { const n = new Set(prev); batch.forEach(b => n.delete(b.path)); return n }) - } - } - - const allSelected = submittablePaths.length > 0 && - submittablePaths.every(p => rows[p]?.selected) - - const toggleSelectAll = () => { - if (!items) return - // Resolved, not-yet-imported units only — a `none` unit with no book can't - // be selected. - const resolvable = items - .map(it => it.path) - .filter(p => rows[p]?.chosen && !results[p]?.accepted) - const turnOn = !resolvable.every(p => rows[p]?.selected) - setRows(prev => { - const n = { ...prev } - for (const p of resolvable) n[p] = { ...n[p], selected: turnOn } - return n - }) - } - - const groups = useMemo(() => { - const g: Record = { confident: [], ambiguous: [], none: [] } - for (const it of items ?? []) g[it.match].push(it) - return g - }, [items]) - - return ( -
-
-

- {t('manualImport.title', 'Manual Import')} -

-

- {t('manualImport.description', 'Scan a folder of files already on disk, match each book to your library, and import them. A file with no match can be added from a metadata search.')} -

-
- -
- setPath(e.target.value)} - onKeyDown={e => { if (e.key === 'Enter') handleScan() }} - placeholder={t('manualImport.pathPlaceholder', '/downloads/books')} - aria-label={t('manualImport.pathLabel', 'Folder to scan')} - className="flex-1 px-3 py-2 rounded-md border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-sm" - /> - -
- - {scanError && ( -

{scanError}

- )} - - {items && items.length === 0 && ( -

- {t('manualImport.empty', 'No book files or folders found here.')} -

- )} - - {items && items.length > 0 && ( - <> - {truncated && ( -

- {t('manualImport.truncated', 'Showing the first 1000 units; narrow the folder to see the rest.')} -

- )} - - {/* Select-all / bulk import bar */} -
- - -
- - {importError && ( -

{importError}

- )} - - {MATCH_ORDER.map(match => { - const group = groups[match] - if (group.length === 0) return null - return ( -
-

- {t(`manualImport.group.${match}`, groupHeading(match))} ({group.length}) -

-
- {group.map(it => ( - pickBook(it.path, b)} - onToggle={() => toggleSelected(it.path)} - onFormat={f => patchRow(it.path, { format: f })} - onImport={() => submit([it.path])} - /> - ))} -
-
- ) - })} - - )} -
- ) -} - -interface ImportRowProps { - item: ScanItem - row: RowState | undefined - result: BatchImportResult | undefined - importing: boolean - onPick: (b: Book) => void - onToggle: () => void - onFormat: (f: string) => void - onImport: () => void -} - -function ImportRow({ item, row, result, importing, onPick, onToggle, onFormat, onImport }: ImportRowProps) { - const { t } = useTranslation() - const [overriding, setOverriding] = useState(false) - const chosen = row?.chosen ?? null - const accepted = result?.accepted - - const badgeCls: Record = { - confident: 'bg-emerald-100 dark:bg-emerald-950 text-emerald-700 dark:text-emerald-400', - ambiguous: 'bg-amber-100 dark:bg-amber-950 text-amber-700 dark:text-amber-400', - none: 'bg-slate-200 dark:bg-zinc-800 text-slate-600 dark:text-zinc-400', - } - - return ( -
-
- -
-
- {item.name} - - {t(`manualImport.group.${item.match}`, groupHeading(item.match))} - - - {item.detectedFormat} - -
- {/* Full source path so the user can tell which file each row refers to; - the basename alone is ambiguous across folders (#1435). */} -
{item.path}
-
- {t('manualImport.parsed', { title: item.parsedTitle || '?', author: item.parsedAuthor || '?', defaultValue: `parsed: ${item.parsedTitle || '?'} / ${item.parsedAuthor || '?'}` })} -
- - {/* Resolution area */} -
- {chosen && !overriding && ( -
- → {bookLabel(chosen)} - {!accepted && ( - - )} -
- )} - - {/* Ambiguous candidate picker */} - {item.match === 'ambiguous' && !chosen && item.candidates && item.candidates.length > 0 && ( -
- {t('manualImport.pickCandidate', 'Pick the correct book')} - {item.candidates.map(c => ( - - ))} -
- )} - - {/* None / override: search the existing catalogue, or add the book - from provider metadata when the library has nothing to bind to. */} - {((item.match === 'none' && !chosen) || overriding) && ( - <> - { onPick(b); setOverriding(false) }} - onCancel={overriding ? () => setOverriding(false) : undefined} - /> - { onPick(b); setOverriding(false) }} - /> - - )} -
-
- -
- - -
-
- - {result && ( -

- {accepted - ? t('manualImport.queued', 'Queued — importing in the background; watch the Queue for progress.') - : t('manualImport.failed', { error: result.error || 'failed', defaultValue: `Failed: ${result.error || 'failed'}` })} -

- )} -
- ) -} - -// BookPicker is a debounced search over the EXISTING catalogue (reuses the same -// /book?search= endpoint FixMatchModal uses). It resolves a `none` unit, or an -// override on a matched one, to a book already in the library. A unit whose -// book is not in the library at all is CatalogueAdder's job, rendered directly -// below this picker. -function BookPicker({ onPick, onCancel }: { onPick: (b: Book) => void; onCancel?: () => void }) { - const { t } = useTranslation() - const [term, setTerm] = useState('') - const [results, setResults] = useState([]) - const [loading, setLoading] = useState(false) - - useEffect(() => { - const q = term.trim() - if (q.length < 2) { - setResults([]) - return - } - let cancelled = false - setLoading(true) - const handle = setTimeout(async () => { - try { - const { items } = await api.listBooks({ search: q, limit: 20 }) - if (!cancelled) setResults(items) - } catch { - if (!cancelled) setResults([]) - } finally { - if (!cancelled) setLoading(false) - } - }, 300) - return () => { cancelled = true; clearTimeout(handle) } - }, [term]) - - return ( -
-
- setTerm(e.target.value)} - placeholder={t('manualImport.searchPlaceholder', 'Search your library for the book…')} - aria-label={t('manualImport.searchLabel', 'Search your library')} - className="flex-1 px-2 py-1 rounded border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-xs" - /> - {onCancel && ( - - )} -
-
- {loading &&

{t('common.loading', 'Loading...')}

} - {!loading && term.trim().length >= 2 && results.length === 0 && ( -

- {t('manualImport.noResults', 'No matching books in your library. Search metadata below to add it.')} -

- )} - {results.map(b => ( - - ))} -
-
- ) -} - -// CatalogueAdder resolves the case BookPicker cannot: the file is for a book -// that is not in the library at all, so there is no row to bind it to (#1719). -// It searches provider metadata (the same ISBN / ASIN / free-text dispatch the -// Add Book modal uses) and creates the picked result through POST /author/book, -// which creates the author too when it is new. The created book is handed back -// so the unit resolves like any other and imports through the existing batch -// call. -// -// searchOnAdd is deliberately false: the user is importing a file they already -// have, so kicking off an indexer search for it would grab a second copy. -function CatalogueAdder({ item, onAdded }: { item: ScanItem; onAdded: (b: Book) => void }) { - const { t } = useTranslation() - const [open, setOpen] = useState(false) - // Prefilled from what the scan already read out of the file, so the user is - // correcting a query rather than retyping one. - const [term, setTerm] = useState(() => [item.parsedTitle, item.parsedAuthor].filter(Boolean).join(' ')) - const [results, setResults] = useState([]) - const [searching, setSearching] = useState(false) - const [searched, setSearched] = useState(false) - const [adding, setAdding] = useState('') - const [error, setError] = useState('') - - if (!open) { - return ( - - ) - } - - const search = async () => { - const q = term.trim() - if (!q) return - setSearching(true) - setError('') - try { - setResults(await resolveBookQuery(q)) - setSearched(true) - } catch (e) { - setResults([]) - setError(e instanceof Error ? e.message : 'Metadata search failed') - } finally { - setSearching(false) - } - } - - const add = async (b: Book) => { - if (!b.foreignBookId || !b.author?.authorName) return - setAdding(b.foreignBookId) - setError('') - try { - const created = await api.addBook({ - foreignBookId: b.foreignBookId, - // May be empty (e.g. a Google Books or DNB result carries a name but no - // author id); the backend resolves the author by name in that case. - foreignAuthorId: b.author.foreignAuthorId ?? '', - authorName: b.author.authorName, - searchOnAdd: false, - }) - onAdded(created) - } catch (e) { - setError(e instanceof Error ? e.message : 'Could not add the book') - } finally { - setAdding('') - } - } - - return ( -
-
- setTerm(e.target.value)} - onKeyDown={e => { if (e.key === 'Enter') search() }} - placeholder={t('manualImport.metadataPlaceholder', 'Title, author, ISBN, or ASIN')} - aria-label={t('manualImport.metadataLabel', 'Search metadata')} - className="flex-1 px-2 py-1 rounded border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-xs" - /> - - -
-

- {t('manualImport.metadataHint', 'Adds the book to your library and links this file to it. No indexer search is started.')} -

- {error &&

{error}

} -
- {!searching && searched && results.length === 0 && !error && ( -

- {t('manualImport.metadataNoResults', 'No metadata results for that search.')} -

- )} - {results.map(b => { - // An author NAME is enough; the backend resolves a missing author id. - // Without one there is nothing to create the book under. - const canAdd = Boolean(b.author?.authorName) - return ( -
-
- {b.title} - {b.author?.authorName && ( - · {b.author.authorName} - )} -
- -
- ) - })} -
-
- ) -} diff --git a/web/src/pages/ManualImportPage.test.tsx b/web/src/pages/import/FolderImportView.test.tsx similarity index 100% rename from web/src/pages/ManualImportPage.test.tsx rename to web/src/pages/import/FolderImportView.test.tsx From ae8293af5c7f44c02a39992e0422cf245551400a Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 02:21:43 -0300 Subject: [PATCH 08/18] feat(web): adopt unmatched library books from the Import page /import becomes one page with two views: "In your library", the new default, and "From a folder" (?view=folder), the existing folder import. The library view has a summary strip with Scan now, a rail of the author folders holding the most undecided books with the one action each calls for, and a list whose rows say in one sentence why the book is here and end in the thing to do. A suggestion is one click to confirm; any row opens an editor in place with the suggestions and their scores, a library search, and a collapsed metadata search that asks a provider only on submit. Decisions show at once with Undo and revert inline on error. Rows are keyboard operable (arrows, Enter, Esc, i, u, /) and stack as cards on narrow screens. Admins see the pending count on the Import nav entry. ManualImportPage is split into ImportPage, FolderImportView and FolderImportRow; BookPicker and CatalogueAdder move to components/import, and SortHeader is extracted. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- web/src/App.test.tsx | 21 ++ web/src/App.tsx | 25 +- web/src/api/adoption.ts | 141 +++++++++++ web/src/api/client.ts | 3 + web/src/components/SortHeader.tsx | 44 ++++ web/src/components/import/BookPicker.tsx | 95 +++++++ web/src/components/import/CatalogueAdder.tsx | 146 +++++++++++ web/src/components/useUnmatchedCount.ts | 33 +++ web/src/i18n/locales/en.json | 135 +++++++++- web/src/pages/import/AdoptionEditor.tsx | 170 +++++++++++++ web/src/pages/import/AdoptionFacets.tsx | 108 ++++++++ web/src/pages/import/AdoptionRail.tsx | 95 +++++++ web/src/pages/import/AdoptionRow.tsx | 171 +++++++++++++ web/src/pages/import/AdoptionSummary.tsx | 92 +++++++ web/src/pages/import/AdoptionTable.tsx | 154 ++++++++++++ web/src/pages/import/AdoptionView.test.tsx | 215 ++++++++++++++++ web/src/pages/import/AdoptionView.tsx | 133 ++++++++++ web/src/pages/import/FolderImportRow.tsx | 161 ++++++++++++ .../pages/import/FolderImportView.test.tsx | 12 +- web/src/pages/import/FolderImportView.tsx | 233 ++++++++++++++++++ web/src/pages/import/ImportPage.test.tsx | 51 ++++ web/src/pages/import/ImportPage.tsx | 97 ++++++++ web/src/pages/import/adoptionHint.test.ts | 51 ++++ web/src/pages/import/adoptionHint.ts | 89 +++++++ web/src/pages/import/adoptionReducer.ts | 112 +++++++++ web/src/pages/import/folderImport.ts | 23 ++ web/src/pages/import/useAdoptionList.ts | Bin 0 -> 6741 bytes 27 files changed, 2599 insertions(+), 11 deletions(-) create mode 100644 web/src/api/adoption.ts create mode 100644 web/src/components/SortHeader.tsx create mode 100644 web/src/components/import/BookPicker.tsx create mode 100644 web/src/components/import/CatalogueAdder.tsx create mode 100644 web/src/components/useUnmatchedCount.ts create mode 100644 web/src/pages/import/AdoptionEditor.tsx create mode 100644 web/src/pages/import/AdoptionFacets.tsx create mode 100644 web/src/pages/import/AdoptionRail.tsx create mode 100644 web/src/pages/import/AdoptionRow.tsx create mode 100644 web/src/pages/import/AdoptionSummary.tsx create mode 100644 web/src/pages/import/AdoptionTable.tsx create mode 100644 web/src/pages/import/AdoptionView.test.tsx create mode 100644 web/src/pages/import/AdoptionView.tsx create mode 100644 web/src/pages/import/FolderImportRow.tsx create mode 100644 web/src/pages/import/FolderImportView.tsx create mode 100644 web/src/pages/import/ImportPage.test.tsx create mode 100644 web/src/pages/import/ImportPage.tsx create mode 100644 web/src/pages/import/adoptionHint.test.ts create mode 100644 web/src/pages/import/adoptionHint.ts create mode 100644 web/src/pages/import/adoptionReducer.ts create mode 100644 web/src/pages/import/folderImport.ts create mode 100644 web/src/pages/import/useAdoptionList.ts diff --git a/web/src/App.test.tsx b/web/src/App.test.tsx index c91551a5a..f629673ae 100644 --- a/web/src/App.test.tsx +++ b/web/src/App.test.tsx @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { render, screen, fireEvent } from '@testing-library/react' import App from './App' +import { api } from './api/client' const { authState, logoutMock } = vi.hoisted(() => ({ authState: { @@ -50,6 +51,8 @@ vi.mock('./api/client', () => ({ // SetupBanner (mounted in the admin shell) probes these on mount. listIndexers: vi.fn().mockResolvedValue([]), listDownloadClients: vi.fn().mockResolvedValue([]), + // The Import nav badge reads this once for an admin. + unmatchedSummary: vi.fn().mockResolvedValue({ pending: 0, pendingFiles: 0, ignored: 0, adopted: 0, scan: {} }), }, })) @@ -282,3 +285,21 @@ describe('Shell — mobile navigation', () => { expect(btn.innerHTML).toContain('M4 6h16M4 12h16M4 18h16') }) }) + +describe('Shell — Import nav badge', () => { + // Admins see how many library books still need a decision on the Import + // entry. The count comes from an admin only route, so nobody else asks. + it('shows the unmatched count for an admin', async () => { + vi.mocked(api.unmatchedSummary).mockResolvedValue({ pending: 38, pendingFiles: 412, ignored: 0, adopted: 0, scan: {} as never }) + authState.value = { status: { authenticated: true, setupRequired: false, mode: 'enabled' }, logout: logoutMock, isAdmin: true } + renderShell() + const desktopNav = document.querySelector('nav.hidden.xl\\:flex') as HTMLElement + const importLink = () => Array.from(desktopNav.querySelectorAll('a')).find(a => a.getAttribute('href') === '/import') + await vi.waitFor(() => expect(importLink()?.textContent).toBe('Import38')) + }) + + it('never asks for the count as a non admin', () => { + renderShell() + expect(api.unmatchedSummary).not.toHaveBeenCalled() + }) +}) diff --git a/web/src/App.tsx b/web/src/App.tsx index 0c15da6ec..7bfc5c168 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -12,6 +12,7 @@ import Logo from './components/Logo' import SetupBanner from './components/SetupBanner' import VersionBadge from './components/VersionBadge' import WhatsNewToast from './components/WhatsNewToast' +import { useUnmatchedCount } from './components/useUnmatchedCount' import { useTheme } from './theme' // Route-scoped error boundary: a render crash in one page shows an inline error @@ -35,7 +36,7 @@ const BooksPage = lazy(() => import('./pages/BooksPage')) const BookDetailPage = lazy(() => import('./pages/BookDetailPage')) const WantedPage = lazy(() => import('./pages/WantedPage')) const QueuePage = lazy(() => import('./pages/QueuePage')) -const ManualImportPage = lazy(() => import('./pages/ManualImportPage')) +const ImportPage = lazy(() => import('./pages/import/ImportPage')) const SettingsPage = lazy(() => import('./pages/SettingsPage')) const UsersPage = lazy(() => import('./pages/UsersPage')) const HistoryPage = lazy(() => import('./pages/HistoryPage')) @@ -109,6 +110,22 @@ function Shell() { const [menuOpen, setMenuOpen] = useState(false) const { status, logout, isAdmin } = useAuth() const signedIn = !!status?.authenticated && status.mode !== 'disabled' + // Books the library scan could not match, on the Import nav entry (admins + // only; the count comes from an admin only route). + const unmatched = useUnmatchedCount(isAdmin) + const navLabel = (key: string) => ( + <> + {t(`nav.${key}`)} + {key === 'import' && unmatched > 0 && ( + + {unmatched > 999 ? '999+' : unmatched} + + )} + + ) useEffect(() => { api.status().then(s => { @@ -140,7 +157,7 @@ function Shell() { @@ -233,7 +250,7 @@ function Shell() { className={mobileLinkClass} onClick={() => setMenuOpen(false)} > - {t(`nav.${item.key}`)} + {navLabel(item.key)} ))} } /> } /> } /> - } /> + } /> } /> } /> } /> diff --git a/web/src/api/adoption.ts b/web/src/api/adoption.ts new file mode 100644 index 000000000..b9d4909ae --- /dev/null +++ b/web/src/api/adoption.ts @@ -0,0 +1,141 @@ +import { request } from './core' + +// Library adoption (#2547): the books a library scan could not match, grouped +// one row per book, and the decisions an admin makes about them. Every route +// is admin only. No request carries a path; rows are addressed by id. + +export type AdoptionState = 'pending' | 'adopted' | 'ignored' + +export type AdoptionReason = + | 'author_not_in_library' + | 'no_candidate_books' + | 'no_title_match' + | 'no_title_parsed' + +export interface AdoptionBookRef { + id: number + title: string + authorId: number + authorName: string + imageUrl?: string + status: string + mediaType: string + monitored: boolean +} + +export interface AdoptionCandidate { + book: AdoptionBookRef + score: number +} + +export interface AdoptionItem { + id: number + kind: 'file' | 'folder' + format: 'ebook' | 'audiobook' + fileCount: number + sizeBytes: number + relPath: string + rootPath: string + authorFolder: string + parsedTitle: string + parsedAuthor: string + reason: AdoptionReason | '' + candidates: AdoptionCandidate[] + topScore: number + state: AdoptionState | 'adopting' | 'undoing' + book?: AdoptionBookRef + bookCreated: boolean + authorCreated: boolean + members: string[] + firstSeenAt: string + resolvedAt?: string +} + +export interface AdoptionFacetCount { + value: string + count: number +} + +export interface AdoptionFolderFacet { + folder: string + units: number + files: number + notInLibrary: number + author: string +} + +export interface AdoptionFacets { + reasons: AdoptionFacetCount[] + formats: AdoptionFacetCount[] + folders: AdoptionFolderFacet[] +} + +export interface AdoptionSummary { + pending: number + pendingFiles: number + ignored: number + adopted: number +} + +export interface AdoptionScanStatus { + ran: boolean + ranAt?: string + running: boolean + filesFound: number + truncated: boolean + error?: string + noFilesFound: boolean +} + +export interface AdoptionListResponse { + items: AdoptionItem[] + total: number + facets?: AdoptionFacets + summary: AdoptionSummary + scan: AdoptionScanStatus +} + +export type AdoptionSort = 'score' | 'title' | 'folder' | 'files' | 'size' | 'seen' + +export interface AdoptionListParams { + state?: AdoptionState + reason?: string + authorFolder?: string + format?: string + search?: string + sort?: AdoptionSort + dir?: 'asc' | 'desc' + limit?: number + offset?: number + facets?: boolean +} + +// What a row is adopted as: a book already in the library, or a metadata +// result the server adds (unmonitored, in the adopted format) first. +export type AdoptTarget = + | { bookId: number; format?: string } + | { foreignBookId: string; foreignAuthorId: string; authorName: string; format?: string } + +export const adoptionApi = { + listUnmatched: (p: AdoptionListParams = {}) => { + const q = new URLSearchParams() + for (const [k, v] of Object.entries(p)) { + if (v === undefined || v === '' || v === false) continue + q.set(k, v === true ? '1' : String(v)) + } + const qs = q.toString() + return request(`/library/unmatched${qs ? `?${qs}` : ''}`) + }, + unmatchedSummary: () => + request('/library/unmatched/summary'), + adoptUnit: (id: number, target: AdoptTarget) => + request(`/library/unmatched/${id}/adopt`, { method: 'POST', body: JSON.stringify(target) }), + undoAdoption: (id: number) => + request(`/library/unmatched/${id}/undo`, { method: 'POST' }), + ignoreUnit: (id: number) => + request(`/library/unmatched/${id}/ignore`, { method: 'POST' }), + unignoreUnit: (id: number) => + request(`/library/unmatched/${id}/unignore`, { method: 'POST' }), + ignoreFolder: (authorFolder: string) => + request<{ ignored: number }>('/library/unmatched/ignore', { method: 'POST', body: JSON.stringify({ authorFolder }) }), +} diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 4413db57f..e468bac55 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -30,6 +30,7 @@ import { grimmoryApi } from './grimmory' import { absApi } from './abs' import { importListsApi } from './importlists' import { recommendationsApi } from './recommendations' +import { adoptionApi } from './adoption' // Shared core: public constant, error class, helpers, and CSRF init. export { ApiError, BINDERY_BASE, isNoDownloadClientError, initCSRF } from './core' @@ -59,6 +60,7 @@ export type * from './grimmory' export type * from './abs' export type * from './importlists' export type * from './recommendations' +export type * from './adoption' // The single `api` object, composed from each domain's slice. Key order mirrors // the original file's section ordering; all method names are unique across @@ -88,4 +90,5 @@ export const api = { ...absApi, ...importListsApi, ...recommendationsApi, + ...adoptionApi, } diff --git a/web/src/components/SortHeader.tsx b/web/src/components/SortHeader.tsx new file mode 100644 index 000000000..34ec9dacf --- /dev/null +++ b/web/src/components/SortHeader.tsx @@ -0,0 +1,44 @@ +import { useTranslation } from 'react-i18next' + +interface Props { + label: string + // The sort key this column sorts by. + sortKey: K + // The active sort key and direction. + activeKey: K + direction: 'asc' | 'desc' + // Clicking the active column flips direction; another column starts at its + // natural direction. + onSort: (key: K) => void + className?: string +} + +// SortHeader is a clickable column header with an arrow for the active sort. +// BooksPage and AuthorDetailPage each carry their own copy; this is the third, +// so it lives here, and moving those two onto it is a separate change. +// +// Tailwind v4's Preflight gives + + ) +} diff --git a/web/src/components/import/BookPicker.tsx b/web/src/components/import/BookPicker.tsx new file mode 100644 index 000000000..3d2b174d4 --- /dev/null +++ b/web/src/components/import/BookPicker.tsx @@ -0,0 +1,95 @@ +import { forwardRef, useEffect, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { api } from '../../api/client' +import type { Book } from '../../api/client' + +interface Props { + onPick: (b: Book) => void + onCancel?: () => void + // Prefills the search, so a person corrects a query instead of typing one. + initialTerm?: string + // Marks one result as the current choice (the adoption editor's radio rows). + selectedId?: number | null +} + +// BookPicker is a debounced search over the EXISTING catalogue (the same +// /book?search= endpoint FixMatchModal uses). It resolves a file to a book +// already in the library. A book that is not in the library at all is +// CatalogueAdder's job. Only the local catalogue is searched here, never a +// metadata provider. +const BookPicker = forwardRef(function BookPicker({ onPick, onCancel, initialTerm = '', selectedId }, ref) { + const { t } = useTranslation() + const [term, setTerm] = useState(initialTerm) + const [results, setResults] = useState([]) + const [loading, setLoading] = useState(false) + + useEffect(() => { + const q = term.trim() + if (q.length < 2) { + setResults([]) + return + } + let cancelled = false + setLoading(true) + const handle = setTimeout(async () => { + try { + const { items } = await api.listBooks({ search: q, limit: 20 }) + if (!cancelled) setResults(items) + } catch { + if (!cancelled) setResults([]) + } finally { + if (!cancelled) setLoading(false) + } + }, 300) + return () => { cancelled = true; clearTimeout(handle) } + }, [term]) + + return ( +
+
+ setTerm(e.target.value)} + placeholder={t('manualImport.searchPlaceholder', 'Search your library for the book…')} + aria-label={t('manualImport.searchLabel', 'Search your library')} + className="flex-1 px-2 py-1 rounded border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-xs" + /> + {onCancel && ( + + )} +
+
+ {loading &&

{t('common.loading', 'Loading...')}

} + {!loading && term.trim().length >= 2 && results.length === 0 && ( +

+ {t('manualImport.noResults', 'No matching books in your library. Search metadata below to add it.')} +

+ )} + {results.map(b => ( + + ))} +
+
+ ) +}) + +export default BookPicker diff --git a/web/src/components/import/CatalogueAdder.tsx b/web/src/components/import/CatalogueAdder.tsx new file mode 100644 index 000000000..5505089bb --- /dev/null +++ b/web/src/components/import/CatalogueAdder.tsx @@ -0,0 +1,146 @@ +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { resolveBookQuery } from '../../api/booklookup' +import type { Book } from '../../api/client' +import { btn, btnSize } from '../buttons' + +interface Props { + // Prefilled from what the scan read out of the file. + initialQuery: string + // Adds the chosen result somehow (add then import, or adopt). A thrown + // error is shown inline. + onChoose: (b: Book) => Promise + // Sentence under the search box saying what choosing a result does. + hint?: string + actionLabel?: string + busyLabel?: string +} + +// CatalogueAdder resolves the case BookPicker cannot: the file is for a book +// that is not in the library at all, so there is no row to bind it to (#1719). +// It searches provider metadata (the same ISBN / ASIN / free-text dispatch the +// Add Book modal uses) and hands the chosen result to onChoose. +// +// It starts collapsed, and a provider is asked only when the person submits +// the search. Opening it, typing and rendering never spend provider quota. +export default function CatalogueAdder({ initialQuery, onChoose, hint, actionLabel, busyLabel }: Props) { + const { t } = useTranslation() + const [open, setOpen] = useState(false) + const [term, setTerm] = useState(initialQuery) + const [results, setResults] = useState([]) + const [searching, setSearching] = useState(false) + const [searched, setSearched] = useState(false) + const [adding, setAdding] = useState('') + const [error, setError] = useState('') + + if (!open) { + return ( + + ) + } + + const search = async () => { + const q = term.trim() + if (!q) return + setSearching(true) + setError('') + try { + setResults(await resolveBookQuery(q)) + setSearched(true) + } catch (e) { + setResults([]) + setError(e instanceof Error ? e.message : 'Metadata search failed') + } finally { + setSearching(false) + } + } + + const choose = async (b: Book) => { + if (!b.foreignBookId || !b.author?.authorName) return + setAdding(b.foreignBookId) + setError('') + try { + await onChoose(b) + } catch (e) { + setError(e instanceof Error ? e.message : 'Could not add the book') + } finally { + setAdding('') + } + } + + return ( +
+
{ e.preventDefault(); void search() }} + > + setTerm(e.target.value)} + placeholder={t('manualImport.metadataPlaceholder', 'Title, author, ISBN, or ASIN')} + aria-label={t('manualImport.metadataLabel', 'Search metadata')} + className="flex-1 px-2 py-1 rounded border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-xs" + /> + + +
+

+ {hint ?? t('manualImport.metadataHint', 'Adds the book to your library and links this file to it. No indexer search is started.')} +

+ {error &&

{error}

} +
+ {!searching && searched && results.length === 0 && !error && ( +

+ {t('manualImport.metadataNoResults', 'No metadata results for that search.')} +

+ )} + {results.map(b => { + // An author NAME is enough; the backend resolves a missing author id. + // Without one there is nothing to create the book under. + const canAdd = Boolean(b.author?.authorName) + return ( +
+
+ {b.title} + {b.author?.authorName && ( + · {b.author.authorName} + )} +
+ +
+ ) + })} +
+
+ ) +} diff --git a/web/src/components/useUnmatchedCount.ts b/web/src/components/useUnmatchedCount.ts new file mode 100644 index 000000000..fe6a8eb6c --- /dev/null +++ b/web/src/components/useUnmatchedCount.ts @@ -0,0 +1,33 @@ +import { useEffect, useState } from 'react' +import { api } from '../api/client' + +// The count behind the Import nav badge: books the library scan could not +// match that still need a decision. Read once per app load for an admin, then +// updated from the event the adoption view fires after each adopt, ignore or +// scan (P5). Nothing polls. + +const EVENT = 'bindery:unmatched-changed' + +export function notifyUnmatchedChanged(pending: number) { + window.dispatchEvent(new CustomEvent(EVENT, { detail: pending })) +} + +export function useUnmatchedCount(enabled: boolean): number { + const [count, setCount] = useState(0) + + useEffect(() => { + if (!enabled) return + let cancelled = false + api.unmatchedSummary() + .then(s => { if (!cancelled) setCount(s.pending) }) + .catch(() => { /* the badge is a convenience; no count shows nothing */ }) + const onChange = (e: Event) => setCount((e as CustomEvent).detail) + window.addEventListener(EVENT, onChange) + return () => { + cancelled = true + window.removeEventListener(EVENT, onChange) + } + }, [enabled]) + + return enabled ? count : 0 +} diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index 612933617..e8a9eacdf 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -80,7 +80,9 @@ "blocklist": "Blocklist", "settings": "Settings", "users": "Users", - "updateAvailable": "Update available" + "updateAvailable": "Update available", + "importUnmatched_one": "{{count}} book needs a decision", + "importUnmatched_other": "{{count}} books need a decision" }, "librarySearch": { "label": "Search library", @@ -140,6 +142,137 @@ "metadataAdd": "Add and use", "metadataAdding": "Adding…" }, + "importPage": { + "title": "Import", + "subtitle": "Bring books you already have into Bindery.", + "viewsLabel": "Import views", + "viewLibrary": "In your library", + "viewFolder": "From a folder" + }, + "adoption": { + "adminOnly": "Only an admin can decide what the files in the library are.", + "searchPlaceholder": "Search titles, authors and folders", + "searchLabel": "Search books to decide", + "stateLabel": "Show", + "state": { + "pending": "Needs a decision", + "ignored": "Ignored", + "adopted": "Adopted" + }, + "filterReason": "Why it did not match", + "filterFormat": "Format", + "chipFolder": "Folder: {{folder}}", + "reason": { + "author_not_in_library": "Author not in your library", + "no_candidate_books": "No book waiting for a file", + "no_title_match": "No close title", + "no_title_parsed": "No title read" + }, + "confirm": "Confirm", + "other": "Other book", + "choose": "Choose book", + "ignore": "Ignore", + "unignore": "Unignore", + "undo": "Undo", + "hint": { + "reasonTooltip": "Scan reason: {{code}}", + "suggestion": "Closest match is {{title}} by {{author}} ({{percent}}%). Confirm it or choose another book.", + "authorMissing": "{{author}} is not in your library yet. Add the author, then scan again.", + "authorUnreadable": "No author could be read from these files. Choose the book they are.", + "noCandidates": "{{author}} has no book in your library waiting for these files. Choose the book or search metadata.", + "noTitle": "No title could be read from the file names or tags. Choose the book they are.", + "noTitleMatch": "No book by {{author}} has a title close to this one. Choose the book it is.", + "noTitleMatchNoAuthor": "No book in your library has a title close to this one. Choose the book it is." + }, + "row": { + "files_one": "{{count}} file", + "files_other": "{{count}} files", + "adoptedAs": "Adopted as" + }, + "outcome": { + "adopting": "Adopting as {{title}}…", + "adopted": "Adopted as {{title}} by {{author}}.", + "created": "Added to your library, unmonitored.", + "ignored": "Ignored. Later scans keep it out of this list.", + "undoing": "Undoing…", + "restored": "Back in Needs a decision." + }, + "editor": { + "heading": "Which book is {{name}}?", + "suggestions": "Suggestions", + "score": "{{percent}}% title match", + "fromSearch": "From search", + "searchLibrary": "Search your library", + "metadataHint": "Adds the book to your library unmonitored, in this format, and adopts these files. Nothing is downloaded.", + "addAndAdopt": "Add and adopt", + "adding": "Adding…", + "format": "Adopt as", + "files_one": "{{count}} file", + "files_other": "{{count}} files", + "moreFiles": "and {{count}} more", + "inPlace": "Files stay where they are. Picking a book already in your library leaves its owner and monitoring as they are.", + "adoptAs": "Adopt as {{title}}", + "pickFirst": "Pick a book" + }, + "col": { + "book": "Book", + "why": "Why it is here", + "match": "Best match" + }, + "caption": { + "pending": "Books in your library that need a decision", + "ignored": "Ignored books in your library", + "adopted": "Books you adopted from your library", + "keys": "Arrow keys move between books, Enter opens one, i ignores, u undoes." + }, + "rail": { + "label": "Folders with the most books to decide", + "heading": "Folders", + "counts_one": "{{count}} book, {{files}} files", + "counts_other": "{{count}} books, {{files}} files", + "notInLibrary": "{{author}} is not in your library", + "addAuthor": "Add author", + "show": "Show", + "showAll": "Show all", + "ignore": "Ignore folder" + }, + "ignoreFolder": { + "title": "Ignore every book in {{folder}}?", + "body": "They leave this list and later scans keep them out. You can unignore any of them from the Ignored list.", + "confirm": "Ignore folder" + }, + "summary": { + "label": "Library scan summary", + "scanning": "Scanning your library…", + "pending_one": "{{count}} book needs a decision, {{files}} files", + "pending_other": "{{count}} books need a decision, {{files}} files", + "none": "Nothing needs a decision", + "lastScan": "Last scan {{when}}", + "neverScanned": "Not scanned yet", + "ignored": "{{count}} ignored", + "adopted": "{{count}} adopted", + "nudge": "Author added. Scan now to match their files.", + "scanningButton": "Scanning…", + "scanNow": "Scan now", + "scanError": "The last scan did not finish: {{error}}", + "noFiles": "The last scan found no book files. Check that the library folder is mounted, then scan again. Your decisions were kept.", + "truncated": "This library is larger than one scan lists. Adopt or ignore some books and scan again to see the rest." + }, + "empty": { + "filteredTitle": "No books match these filters", + "filteredBody": "Try another search, or clear the filters to see every book.", + "clear": "Clear filters", + "ignoredTitle": "Nothing is ignored", + "ignoredBody": "Books you ignore land here, and you can bring any of them back.", + "adoptedTitle": "Nothing adopted yet", + "adoptedBody": "Books you adopt stay here for 30 days, with Undo.", + "neverTitle": "Your library has not been scanned yet", + "neverBody": "A scan looks through your library folders and lists every book it cannot match to your library, so you can decide what each one is. Files are never moved.", + "scanLibrary": "Scan library", + "doneTitle": "Every book in your library is matched", + "doneBody": "Nothing needs a decision. New files show up here after the next scan." + } + }, "search": { "heading": "Search Indexers", "placeholder": "Title, author, or keyword…", diff --git a/web/src/pages/import/AdoptionEditor.tsx b/web/src/pages/import/AdoptionEditor.tsx new file mode 100644 index 000000000..058a7eb3e --- /dev/null +++ b/web/src/pages/import/AdoptionEditor.tsx @@ -0,0 +1,170 @@ +import { useEffect, useId, useRef, useState, type KeyboardEvent } from 'react' +import { useTranslation } from 'react-i18next' +import type { AdoptionBookRef, AdoptionItem, AdoptTarget, Book } from '../../api/client' +import { btn, btnSize } from '../../components/buttons' +import BookPicker from '../../components/import/BookPicker' +import CatalogueAdder from '../../components/import/CatalogueAdder' +import { scorePercent, unitDisplayName } from './adoptionHint' + +interface Props { + item: AdoptionItem + onAdopt: (target: AdoptTarget, preview: AdoptionBookRef | null) => void | Promise + onCancel: () => void +} + +function refFromBook(b: Book): AdoptionBookRef { + return { + id: b.id, title: b.title, authorId: b.authorId, authorName: b.author?.authorName ?? '', + imageUrl: b.imageUrl, status: b.status, mediaType: b.mediaType ?? '', monitored: b.monitored, + } +} + +// AdoptionEditor opens in place under its row, never as a dialog: the +// suggestions as radio rows with their scores, a search of the library, +// prefilled and focused, and a collapsed metadata search that asks a provider +// only when submitted. Esc closes it and focus goes back to the row. +export default function AdoptionEditor({ item, onAdopt, onCancel }: Props) { + const { t } = useTranslation() + const headingId = useId() + const searchRef = useRef(null) + const [chosen, setChosen] = useState(item.candidates[0]?.book ?? null) + const [searched, setSearched] = useState(null) + const [format, setFormat] = useState(item.format) + const name = unitDisplayName(item) + + useEffect(() => { searchRef.current?.focus() }, []) + + const onKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Escape') { + e.stopPropagation() + onCancel() + } + } + const pickSearched = (b: Book) => { + const ref = refFromBook(b) + setSearched(ref) + setChosen(ref) + } + const options: { book: AdoptionBookRef; score?: number }[] = [ + ...item.candidates, + ...(searched && !item.candidates.some(c => c.book.id === searched.id) ? [{ book: searched }] : []), + ] + const formatOverride = format !== item.format ? format : undefined + const more = item.fileCount - item.members.length + + return ( +
+

+ {t('adoption.editor.heading', { name, defaultValue: 'Which book is {{name}}?' })} +

+ + {options.length > 0 && ( +
+ + {t('adoption.editor.suggestions', 'Suggestions')} + +
+ {options.map(o => { + const pct = o.score === undefined ? null : scorePercent(o.score) + return ( + + ) + })} +
+
+ )} + +
+
+

{t('adoption.editor.searchLibrary', 'Search your library')}

+ + { + await onAdopt({ + foreignBookId: b.foreignBookId, + foreignAuthorId: b.author?.foreignAuthorId ?? '', + authorName: b.author?.authorName ?? '', + format: formatOverride, + }, null) + }} + /> +
+
+
+

{t('adoption.editor.format', 'Adopt as')}

+
+ {(['ebook', 'audiobook'] as const).map(f => ( + + ))} +
+
+ {item.members.length > 1 && ( +
+ + {t('adoption.editor.files', { count: item.fileCount, defaultValue: '{{count}} files' })} + +
    + {item.members.map(m =>
  • {m}
    • )} + {more > 0 &&
  • {t('adoption.editor.moreFiles', { count: more, defaultValue: 'and {{count}} more' })}
    • } +
+
+ )} +

+ {t('adoption.editor.inPlace', 'Files stay where they are. Picking a book already in your library leaves its owner and monitoring as they are.')} +

+
+
+ +
+ + +
+
+ ) +} diff --git a/web/src/pages/import/AdoptionFacets.tsx b/web/src/pages/import/AdoptionFacets.tsx new file mode 100644 index 000000000..4f9c48e60 --- /dev/null +++ b/web/src/pages/import/AdoptionFacets.tsx @@ -0,0 +1,108 @@ +import { forwardRef } from 'react' +import { useTranslation } from 'react-i18next' +import type { AdoptionFacets as Facets, AdoptionState, AdoptionSummary } from '../../api/client' +import FilterPopover, { FilterGroup } from '../../components/FilterPopover' +import type { AdoptionFilters } from './useAdoptionList' + +interface Props { + filters: AdoptionFilters + facets: Facets | null + summary: AdoptionSummary | null + onChange: (patch: Partial) => void +} + +const STATES: AdoptionState[] = ['pending', 'ignored', 'adopted'] + +// The toolbar over the list: search, which decisions to look at, and the +// reason and format facets behind one disclosure. An applied filter is always +// visible as a removable chip, the same rule BooksPage follows. +const AdoptionFacets = forwardRef(function AdoptionFacets({ filters, facets, summary, onChange }, searchRef) { + const { t } = useTranslation() + + const stateLabel = (s: AdoptionState) => { + const count = s === 'pending' ? summary?.pending : s === 'ignored' ? summary?.ignored : summary?.adopted + const label = t(`adoption.state.${s}`, s === 'pending' ? 'Needs a decision' : s === 'ignored' ? 'Ignored' : 'Adopted') + return count ? `${label} (${count})` : label + } + const reasonLabel = (code: string) => t(`adoption.reason.${code}`, code) + const formatLabel = (f: string) => (f === 'audiobook' ? t('common.audiobook', 'Audiobook') : t('common.ebook', 'Ebook')) + const pill = (active: boolean) => + `px-3 py-1 rounded-md text-xs font-medium transition-colors ${active ? 'bg-slate-300 dark:bg-zinc-700 text-slate-900 dark:text-white' : 'text-slate-600 dark:text-zinc-400 hover:text-slate-900 dark:hover:text-white'}` + const chip = 'inline-flex items-center gap-1 px-2 py-1 rounded-full bg-slate-200 dark:bg-zinc-800 text-xs text-slate-700 dark:text-zinc-300 hover:bg-slate-300 dark:hover:bg-zinc-700' + + return ( +
+
+ onChange({ search: e.target.value })} + placeholder={t('adoption.searchPlaceholder', 'Search titles, authors and folders')} + aria-label={t('adoption.searchLabel', 'Search books to decide')} + aria-keyshortcuts="/" + className="flex-1 bg-slate-200 dark:bg-zinc-800 border border-slate-300 dark:border-zinc-700 rounded px-3 py-2 text-sm focus:outline-none focus:border-slate-400 dark:focus:border-zinc-600 placeholder-slate-400 dark:placeholder-zinc-600" + /> +
+ {STATES.map(s => ( + + ))} +
+
+
+ + onChange({ reason })} + options={[ + { value: '', label: t('common.all', 'All') }, + ...(facets?.reasons ?? []).map(r => ({ value: r.value, label: `${reasonLabel(r.value)} (${r.count})` })), + ]} + /> + onChange({ format })} + options={[ + { value: '', label: t('common.all', 'All') }, + ...(facets?.formats ?? []).map(f => ({ value: f.value, label: `${formatLabel(f.value)} (${f.count})` })), + ]} + /> + + {filters.authorFolder && ( + + )} + {filters.reason && ( + + )} + {filters.format && ( + + )} +
+
+ ) +}) + +export default AdoptionFacets diff --git a/web/src/pages/import/AdoptionRail.tsx b/web/src/pages/import/AdoptionRail.tsx new file mode 100644 index 000000000..712fbf359 --- /dev/null +++ b/web/src/pages/import/AdoptionRail.tsx @@ -0,0 +1,95 @@ +import { useTranslation } from 'react-i18next' +import type { AdoptionFolderFacet } from '../../api/client' + +interface Props { + folders: AdoptionFolderFacet[] + activeFolder: string + onShow: (folder: string) => void + onAddAuthor: (name: string) => void + onIgnoreFolder: (folder: string) => void +} + +// The structure rail: the author folders holding the most undecided books, +// each with the one action its books call for. When every book in a folder +// failed because its author is not in the library, that is one decision for +// the whole folder (add the author, then scan), not a decision per file. +// Anything else is a folder worth looking at, so it filters the list. +// +// A column beside the list on wide screens, a row of scrolling chips on narrow +// ones. +export default function AdoptionRail({ folders, activeFolder, onShow, onAddAuthor, onIgnoreFolder }: Props) { + const { t } = useTranslation() + if (folders.length === 0) return null + + return ( + + ) +} diff --git a/web/src/pages/import/AdoptionRow.tsx b/web/src/pages/import/AdoptionRow.tsx new file mode 100644 index 000000000..f60232344 --- /dev/null +++ b/web/src/pages/import/AdoptionRow.tsx @@ -0,0 +1,171 @@ +import { forwardRef, type KeyboardEvent } from 'react' +import { Link } from 'react-router' +import { useTranslation } from 'react-i18next' +import type { AdoptionItem } from '../../api/client' +import { btn, btnSize } from '../../components/buttons' +import MediaBadge from '../../components/MediaBadge' +import { formatBytes } from '../../util/format' +import { adoptionHint, scorePercent, unitDisplayName } from './adoptionHint' +import type { Outcome } from './adoptionReducer' + +interface Props { + item: AdoptionItem + outcome: Outcome | undefined + error: string | undefined + expanded: boolean + focusable: boolean + editorId: string + onFocusRow: () => void + onKeyDown: (e: KeyboardEvent) => void + onToggle: () => void + onConfirm: () => void + onIgnore: () => void + onUndo: () => void + onAddAuthor: (name: string) => void +} + +// A score pill reads green when the suggestion is close, amber when it is a +// real question. +function scoreCls(percent: number): string { + return percent >= 80 + ? 'bg-emerald-100 text-emerald-800 dark:bg-emerald-950 dark:text-emerald-300' + : 'bg-amber-100 text-amber-800 dark:bg-amber-950 dark:text-amber-300' +} + +// AdoptionRow is one book in the collapsed list: what it is, a sentence about +// why it is here, and the action that most likely settles it. With a +// suggestion that is a single Confirm. A decision replaces the sentence and +// actions with a quiet line that keeps Undo until the next fetch. +const AdoptionRow = forwardRef(function AdoptionRow( + { item, outcome, error, expanded, focusable, editorId, onFocusRow, onKeyDown, onToggle, onConfirm, onIgnore, onUndo, onAddAuthor }, ref, +) { + const { t } = useTranslation() + const name = unitDisplayName(item) + const hint = adoptionHint(item, t) + const top = item.candidates[0] + const size = formatBytes(item.sizeBytes) + const cell = 'block md:table-cell px-3 py-1 md:py-2.5 align-top' + + const outcomeLine = (() => { + if (!outcome) return null + switch (outcome.kind) { + case 'adopting': + return t('adoption.outcome.adopting', { title: outcome.preview?.title ?? name, defaultValue: 'Adopting as {{title}}…' }) + case 'adopted': { + const book = outcome.item.book + const base = t('adoption.outcome.adopted', { title: book?.title ?? name, author: book?.authorName ?? '', defaultValue: 'Adopted as {{title}} by {{author}}.' }) + return outcome.item.bookCreated ? `${base} ${t('adoption.outcome.created', 'Added to your library, unmonitored.')}` : base + } + case 'ignoring': + case 'ignored': + return t('adoption.outcome.ignored', 'Ignored. Later scans keep it out of this list.') + case 'undoing': + return t('adoption.outcome.undoing', 'Undoing…') + case 'restored': + return t('adoption.outcome.restored', 'Back in Needs a decision.') + } + })() + const canUndo = outcome?.kind === 'adopted' || outcome?.kind === 'ignored' + // When the scan could not place the author at all, adding the author is the + // decision; choosing a book stays available beside it. + const addAuthorFirst = item.state === 'pending' && !top && item.reason === 'author_not_in_library' && item.parsedAuthor !== '' + + return ( + { if (e.target === e.currentTarget) onFocusRow() }} + onKeyDown={onKeyDown} + aria-label={name} + className={`block md:table-row mb-3 md:mb-0 rounded-lg md:rounded-none border md:border-0 border-slate-200 dark:border-zinc-800 outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-emerald-500 ${ + expanded ? 'bg-emerald-500/5' : 'bg-slate-100/50 dark:bg-zinc-900/50 hover:bg-slate-200/50 dark:hover:bg-zinc-800/50' + }`} + > + +
+ {name} + + + {t('adoption.row.files', { count: item.fileCount, defaultValue: '{{count}} files' })}{size ? ` · ${size}` : ''} + +
+
+ {item.relPath} +
+ + + {outcomeLine ? ( + +

+ {outcome?.kind === 'adopted' || outcome?.kind === 'ignored' ? '✓ ' : ''}{outcomeLine} + {canUndo && ( + + )} +

+ + ) : ( + <> + +

+ {item.state === 'adopted' && item.book ? ( + <> + {t('adoption.row.adoptedAs', 'Adopted as')}{' '} + {item.book.title} + {item.book.authorName ? ` · ${item.book.authorName}` : ''} + + ) : hint.sentence} +

+ {error &&

{error}

} + + +
+ {item.state === 'pending' && top && ( + <> + + {top.book.title} + + + {scorePercent(top.score)}% + + + + )} + {addAuthorFirst && ( + + )} + {item.state === 'pending' && ( + + )} + {item.state === 'pending' && ( + + )} + {(item.state === 'ignored' || item.state === 'adopted') && ( + + )} +
+ + + )} + + ) +}) + +export default AdoptionRow diff --git a/web/src/pages/import/AdoptionSummary.tsx b/web/src/pages/import/AdoptionSummary.tsx new file mode 100644 index 000000000..c140315b7 --- /dev/null +++ b/web/src/pages/import/AdoptionSummary.tsx @@ -0,0 +1,92 @@ +import { useTranslation } from 'react-i18next' +import type { AdoptionScanStatus, AdoptionSummary as Summary } from '../../api/client' +import { btn, btnSize } from '../../components/buttons' +import { relativeTime } from './adoptionHint' + +interface Props { + summary: Summary | null + scan: AdoptionScanStatus | null + onScan: () => void + // Set after an author is added from the rail: the next useful step is a scan. + nudgeScan: boolean +} + +// The strip above the list: how much is left to decide, when the library was +// last looked at, and the one control that refreshes it. +export default function AdoptionSummary({ summary, scan, onScan, nudgeScan }: Props) { + const { t, i18n } = useTranslation() + const running = Boolean(scan?.running) + const pending = summary?.pending ?? 0 + + let headline: string + if (running) headline = t('adoption.summary.scanning', 'Scanning your library…') + else if (pending > 0) { + headline = t('adoption.summary.pending', { + count: pending, + files: summary?.pendingFiles ?? 0, + defaultValue: '{{count}} books need a decision, {{files}} files', + }) + } else headline = t('adoption.summary.none', 'Nothing needs a decision') + + const ranAt = scan?.ran ? relativeTime(scan.ranAt, i18n.language) : '' + + return ( +
+
+
+

+ {running && ( +

+

+ {ranAt + ? t('adoption.summary.lastScan', { when: ranAt, defaultValue: 'Last scan {{when}}' }) + : t('adoption.summary.neverScanned', 'Not scanned yet')} + {summary && summary.ignored > 0 && ( + <> · {t('adoption.summary.ignored', { count: summary.ignored, defaultValue: '{{count}} ignored' })} + )} + {summary && summary.adopted > 0 && ( + <> · {t('adoption.summary.adopted', { count: summary.adopted, defaultValue: '{{count}} adopted' })} + )} +

+
+ {nudgeScan && !running && ( +

+ {t('adoption.summary.nudge', 'Author added. Scan now to match their files.')} +

+ )} + +
+ {scan?.error && ( +

+ {t('adoption.summary.scanError', { error: scan.error, defaultValue: 'The last scan did not finish: {{error}}' })} +

+ )} + {!scan?.error && scan?.ran && scan.noFilesFound && ( +

+ {t('adoption.summary.noFiles', 'The last scan found no book files. Check that the library folder is mounted, then scan again. Your decisions were kept.')} +

+ )} + {scan?.truncated && ( +

+ {t('adoption.summary.truncated', 'This library is larger than one scan lists. Adopt or ignore some books and scan again to see the rest.')} +

+ )} +
+ ) +} diff --git a/web/src/pages/import/AdoptionTable.tsx b/web/src/pages/import/AdoptionTable.tsx new file mode 100644 index 000000000..25a71660b --- /dev/null +++ b/web/src/pages/import/AdoptionTable.tsx @@ -0,0 +1,154 @@ +import { Fragment, useEffect, useRef, useState, type KeyboardEvent } from 'react' +import { useTranslation } from 'react-i18next' +import type { AdoptionItem, AdoptionSort } from '../../api/client' +import SortHeader from '../../components/SortHeader' +import AdoptionEditor from './AdoptionEditor' +import AdoptionRow from './AdoptionRow' +import type { AdoptionList } from './useAdoptionList' + +interface Props { + list: AdoptionList + onSearchShortcut: () => void + onAddAuthor: (name: string) => void +} + +// Natural direction per column: scores, file counts and sizes read biggest +// first, names read A to Z. +const NATURAL_DESC: Record = { score: true, files: true, size: true, seen: true, title: false, folder: false } + +// AdoptionTable is the list itself: a real table on wide screens, stacked +// cards on narrow ones, operable from the keyboard. One row holds the tab stop +// (roving tabindex); arrows move it, Enter opens the row's editor in place, +// Esc closes it and returns focus to the row, i ignores, u undoes, and / +// jumps to the search box. +export default function AdoptionTable({ list, onSearchShortcut, onAddAuthor }: Props) { + const { t } = useTranslation() + const { state, filters } = list + const [focusIndex, setFocusIndex] = useState(0) + const rowRefs = useRef<(HTMLTableRowElement | null)[]>([]) + const restoreFocusTo = useRef(null) + const items = state.items + const direction: 'asc' | 'desc' = filters.dir || (NATURAL_DESC[filters.sort] ? 'desc' : 'asc') + + useEffect(() => { setFocusIndex(i => Math.min(i, Math.max(0, items.length - 1))) }, [items.length]) + + // After the editor closes, focus goes back to the row it belonged to. + useEffect(() => { + if (state.expandedId === null && restoreFocusTo.current !== null) { + rowRefs.current[restoreFocusTo.current]?.focus() + restoreFocusTo.current = null + } + }, [state.expandedId]) + + const sortBy = (key: AdoptionSort) => { + if (key === filters.sort) list.setFilter({ dir: direction === 'asc' ? 'desc' : 'asc' }) + else list.setFilter({ sort: key, dir: '' }) + } + + const closeEditor = (index: number) => { + restoreFocusTo.current = index + list.collapse() + } + + const confirm = (item: AdoptionItem) => { + const top = item.candidates[0] + if (top) void list.adopt(item, { bookId: top.book.id }, top.book) + } + + const onRowKey = (e: KeyboardEvent, index: number, item: AdoptionItem) => { + if (e.target !== e.currentTarget) return + const outcome = state.outcomes[item.id] + const move = (to: number) => { + e.preventDefault() + const next = Math.max(0, Math.min(items.length - 1, to)) + setFocusIndex(next) + rowRefs.current[next]?.focus() + } + switch (e.key) { + case 'ArrowDown': return move(index + 1) + case 'ArrowUp': return move(index - 1) + case 'Home': return move(0) + case 'End': return move(items.length - 1) + case 'Enter': + e.preventDefault() + if (outcome || item.state !== 'pending') return + if (state.expandedId === item.id) closeEditor(index) + else list.expand(item.id) + return + case 'Escape': + if (state.expandedId === item.id) closeEditor(index) + return + case 'i': + if (!outcome && item.state === 'pending') { e.preventDefault(); void list.ignore(item) } + return + case 'u': + if (outcome?.kind === 'adopted' || outcome?.kind === 'ignored' || (!outcome && item.state !== 'pending')) { + e.preventDefault() + void list.undo(item) + } + return + case '/': + e.preventDefault() + onSearchShortcut() + } + } + + const captionKey = filters.state === 'ignored' ? 'adoption.caption.ignored' : filters.state === 'adopted' ? 'adoption.caption.adopted' : 'adoption.caption.pending' + const captionDefault = filters.state === 'ignored' ? 'Ignored books in your library' : filters.state === 'adopted' ? 'Books you adopted from your library' : 'Books in your library that need a decision' + + return ( +
+ + + + + + + + + + + {items.map((item, index) => { + const expanded = state.expandedId === item.id + const editorId = `adoption-editor-${item.id}` + return ( + + { rowRefs.current[index] = el }} + item={item} + outcome={state.outcomes[item.id]} + error={state.errors[item.id]} + expanded={expanded} + focusable={index === focusIndex} + editorId={editorId} + onFocusRow={() => setFocusIndex(index)} + onKeyDown={e => onRowKey(e, index, item)} + onToggle={() => (expanded ? closeEditor(index) : list.expand(item.id))} + onConfirm={() => confirm(item)} + onIgnore={() => void list.ignore(item)} + onUndo={() => void list.undo(item)} + onAddAuthor={onAddAuthor} + /> + {expanded && ( + + + + )} + + ) + })} + +
+ {t(captionKey, captionDefault)}. {t('adoption.caption.keys', 'Arrow keys move between books, Enter opens one, i ignores, u undoes.')} +
+ {t('adoption.col.why', 'Why it is here')} +
+ list.adopt(item, target, preview)} + onCancel={() => closeEditor(index)} + /> +
+
+ ) +} diff --git a/web/src/pages/import/AdoptionView.test.tsx b/web/src/pages/import/AdoptionView.test.tsx new file mode 100644 index 000000000..5153eb81d --- /dev/null +++ b/web/src/pages/import/AdoptionView.test.tsx @@ -0,0 +1,215 @@ +import { fireEvent, render, screen, waitFor, within } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { http, HttpResponse } from 'msw' +import { MemoryRouter } from 'react-router' +import { apiUrl, server } from '../../test/msw' +import type { AdoptionItem, AdoptionListResponse } from '../../api/client' +import AdoptionView from './AdoptionView' + +// AdoptionView talks to the server through the real api client, mocked at the +// network with MSW (T1), so a fetch spy sees exactly what the page requests. +// Strings render from en.json, so the assertions read like the page does. +vi.mock('react-i18next', async () => { + const en = (await import('../../i18n/locales/en.json')).default as Record + const lookup = (key: string): unknown => + key.split('.').reduce((n, p) => (n && typeof n === 'object' ? (n as Record)[p] : undefined), en) + const t = (key: string, options?: string | Record) => { + const opts = typeof options === 'object' ? options : {} + let value = lookup(key) + if (typeof opts.count === 'number') value = lookup(`${key}_${opts.count === 1 ? 'one' : 'other'}`) ?? value + if (typeof value !== 'string') value = typeof options === 'string' ? options : (opts.defaultValue as string) ?? key + return (value as string).replace(/\{\{(\w+)\}\}/g, (_, k) => String(opts[k] ?? '')) + } + return { useTranslation: () => ({ t, i18n: { language: 'en' } }) } +}) + +function item(overrides: Partial & Pick): AdoptionItem { + return { + kind: 'file', format: 'ebook', fileCount: 1, sizeBytes: 2048, relPath: `Andy Weir/Book ${overrides.id}.epub`, + rootPath: '/books', authorFolder: 'Andy Weir', parsedTitle: `Book ${overrides.id}`, parsedAuthor: 'Andy Weir', + reason: 'no_title_match', candidates: [], topScore: 0, state: 'pending', bookCreated: false, authorCreated: false, + members: [`Book ${overrides.id}.epub`], firstSeenAt: '2026-09-17T00:00:00Z', + ...overrides, + } +} + +const martian = { id: 42, title: 'The Martian', authorId: 7, authorName: 'Andy Weir', status: 'wanted', mediaType: 'ebook', monitored: true } + +function listResponse(items: AdoptionItem[], overrides: Partial = {}): AdoptionListResponse { + return { + items, + total: items.length, + facets: { reasons: [{ value: 'no_title_match', count: items.length }], formats: [{ value: 'ebook', count: items.length }], folders: [] }, + summary: { pending: items.length, pendingFiles: items.length, ignored: 0, adopted: 0 }, + scan: { ran: true, ranAt: new Date(Date.now() - 2 * 3600_000).toISOString(), running: false, filesFound: 10, truncated: false, noFilesFound: false }, + ...overrides, + } +} + +const suggested = item({ + id: 1, parsedTitle: 'A Martyrs Tale', relPath: 'Andy Weir/A Martyrs Tale.epub', + candidates: [{ book: martian, score: 0.82 }], topScore: 0.82, +}) +const unsuggested = item({ id: 2, parsedTitle: 'Mystery Notes', relPath: 'Andy Weir/Mystery Notes.epub' }) + +let fetchSpy: { mock: { calls: Parameters[] }; mockRestore: () => void } + +function requestedURLs(): string[] { + return fetchSpy.mock.calls.map(([input]: Parameters) => (typeof input === 'string' ? input : input instanceof URL ? input.href : (input as Request).url)) +} + +function serve(response: AdoptionListResponse) { + server.use( + http.get(apiUrl('/library/unmatched'), () => HttpResponse.json(response)), + http.get(apiUrl('/library/unmatched/summary'), () => HttpResponse.json({ ...response.summary, scan: response.scan })), + ) +} + +async function renderView() { + render() + return screen.findByRole('table', { name: /Books in your library that need a decision/ }) +} + +beforeEach(() => { + localStorage.clear() + fetchSpy = vi.spyOn(window, 'fetch') +}) + +afterEach(() => { + fetchSpy.mockRestore() +}) + +describe('AdoptionView', () => { + it('confirms a suggestion in one click, then undoes it', async () => { + serve(listResponse([suggested, unsuggested])) + let adoptBody: unknown = null + server.use( + http.post(apiUrl('/library/unmatched/1/adopt'), async ({ request }) => { + adoptBody = await request.json() + return HttpResponse.json({ ...suggested, state: 'adopted', book: martian }) + }), + http.post(apiUrl('/library/unmatched/1/undo'), () => HttpResponse.json(suggested)), + ) + const table = await renderView() + + const row = within(table).getByRole('row', { name: 'A Martyrs Tale' }) + expect(within(row).getByText(/Closest match is The Martian by Andy Weir \(82%\)/)).toBeInTheDocument() + fireEvent.click(within(row).getByRole('button', { name: 'Confirm' })) + + expect(await within(row).findByText(/Adopted as The Martian by Andy Weir\./)).toBeInTheDocument() + expect(adoptBody).toEqual({ bookId: 42 }) + + fireEvent.click(within(row).getByRole('button', { name: 'Undo' })) + expect(await within(row).findByRole('button', { name: 'Confirm' })).toBeInTheDocument() + }) + + it('reverts an optimistic adopt and shows the error on the row', async () => { + serve(listResponse([suggested])) + server.use( + http.post(apiUrl('/library/unmatched/1/adopt'), () => + HttpResponse.json({ error: 'Provenance.epub already belongs to a book in your library.' }, { status: 409 })), + ) + const table = await renderView() + const row = within(table).getByRole('row', { name: 'A Martyrs Tale' }) + fireEvent.click(within(row).getByRole('button', { name: 'Confirm' })) + + expect(await within(row).findByRole('alert')).toHaveTextContent('already belongs to a book') + expect(within(row).getByRole('button', { name: 'Confirm' })).toBeInTheDocument() + }) + + it('asks a metadata provider only when the search is submitted', async () => { + serve(listResponse([unsuggested])) + server.use(http.get(apiUrl('/search/book'), () => HttpResponse.json([]))) + const table = await renderView() + + const providerCalls = () => requestedURLs().filter(u => u.includes('/search/book') || u.includes('/book/lookup')) + fireEvent.click(within(table).getByRole('button', { name: 'Choose book' })) + await screen.findByRole('region', { name: 'Which book is Mystery Notes?' }) + fireEvent.click(screen.getByRole('button', { name: 'Not in your library? Search metadata' })) + fireEvent.change(screen.getByRole('textbox', { name: 'Search metadata' }), { target: { value: 'mystery notes weir' } }) + expect(providerCalls()).toEqual([]) + + fireEvent.click(screen.getByRole('button', { name: 'Search' })) + await waitFor(() => expect(providerCalls()).toHaveLength(1)) + expect(providerCalls()[0]).toContain('/search/book?term=mystery%20notes%20weir') + }) + + it('is operable from the keyboard: arrows, Enter, Esc and i', async () => { + serve(listResponse([suggested, unsuggested])) + server.use(http.get(apiUrl('/book'), () => HttpResponse.json({ items: [], total: 0 }))) + let ignored = false + server.use(http.post(apiUrl('/library/unmatched/2/ignore'), () => { + ignored = true + return HttpResponse.json({ ...unsuggested, state: 'ignored' }) + })) + const table = await renderView() + const [first, second] = within(table).getAllByRole('row').slice(1) + + expect(first).toHaveAttribute('tabindex', '0') + expect(second).toHaveAttribute('tabindex', '-1') + first.focus() + fireEvent.keyDown(first, { key: 'ArrowDown' }) + expect(second).toHaveFocus() + expect(second).toHaveAttribute('tabindex', '0') + + fireEvent.keyDown(second, { key: 'Enter' }) + const editor = await screen.findByRole('region', { name: 'Which book is Mystery Notes?' }) + expect(within(editor).getByRole('textbox', { name: 'Search your library' })).toHaveFocus() + + fireEvent.keyDown(within(editor).getByRole('textbox', { name: 'Search your library' }), { key: 'Escape' }) + await waitFor(() => expect(screen.queryByRole('region', { name: /Which book is/ })).toBeNull()) + expect(second).toHaveFocus() + + fireEvent.keyDown(second, { key: 'i' }) + expect(await within(second).findByText(/Ignored\. Later scans keep it out of this list\./)).toBeInTheDocument() + expect(ignored).toBe(true) + }) + + it('shows the never scanned state with a scan action', async () => { + serve(listResponse([], { scan: { ran: false, running: false, filesFound: 0, truncated: false, noFilesFound: false } })) + render() + expect(await screen.findByText('Your library has not been scanned yet')).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Scan library' })).toBeInTheDocument() + expect(screen.getByText('Not scanned yet')).toBeInTheDocument() + }) + + it('shows the all matched state after a scan with nothing left', async () => { + serve(listResponse([])) + render() + expect(await screen.findByText('Every book in your library is matched')).toBeInTheDocument() + expect(screen.getByText('Nothing needs a decision')).toBeInTheDocument() + }) + + it('says when the scan was truncated and when one is running', async () => { + serve(listResponse([unsuggested], { + scan: { ran: true, ranAt: new Date().toISOString(), running: true, filesFound: 60000, truncated: true, noFilesFound: false }, + })) + await renderView() + expect(screen.getByText('Scanning your library…')).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Scanning…' })).toBeDisabled() + expect(screen.getByText(/larger than one scan lists/)).toBeInTheDocument() + }) + + it('lists ignored books with Unignore and an empty ignored state', async () => { + serve(listResponse([])) + render() + await screen.findByText('Every book in your library is matched') + + const ignoredItem = item({ id: 3, parsedTitle: 'Old Manual', state: 'ignored' }) + server.use(http.get(apiUrl('/library/unmatched'), ({ request }) => + HttpResponse.json(new URL(request.url).searchParams.get('state') === 'ignored' ? listResponse([ignoredItem]) : listResponse([])))) + fireEvent.click(screen.getByRole('radio', { name: /Ignored/ })) + const table = await screen.findByRole('table', { name: /Ignored books in your library/ }) + expect(within(table).getByRole('button', { name: 'Unignore' })).toBeInTheDocument() + }) + + it('offers Add author for a folder whose author is not in the library', async () => { + serve(listResponse([unsuggested], { + facets: { reasons: [], formats: [], folders: [{ folder: 'Becky Chambers', units: 4, files: 40, notInLibrary: 4, author: 'Becky Chambers' }] }, + })) + await renderView() + const rail = screen.getByRole('navigation', { name: 'Folders with the most books to decide' }) + expect(within(rail).getByText('Becky Chambers is not in your library')).toBeInTheDocument() + expect(within(rail).getByRole('button', { name: 'Add author' })).toBeInTheDocument() + }) +}) diff --git a/web/src/pages/import/AdoptionView.tsx b/web/src/pages/import/AdoptionView.tsx new file mode 100644 index 000000000..44692d6fd --- /dev/null +++ b/web/src/pages/import/AdoptionView.tsx @@ -0,0 +1,133 @@ +import { useEffect, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import AddToLibraryModal from '../../components/AddToLibraryModal' +import { btn, btnSize } from '../../components/buttons' +import Pagination from '../../components/Pagination' +import { useConfirmDialog } from '../../components/useConfirmDialog' +import AdoptionFacets from './AdoptionFacets' +import AdoptionRail from './AdoptionRail' +import AdoptionSummary from './AdoptionSummary' +import AdoptionTable from './AdoptionTable' +import { defaultAdoptionFilters, useAdoptionList } from './useAdoptionList' + +// AdoptionView is the Import page's "In your library" view. It lays out the +// summary strip, the folder rail, the toolbar and the list, and chooses which +// designed state to show when there is no list: never scanned, all matched, +// nothing ignored or adopted yet, or nothing matching the filters. The data +// and every decision live in useAdoptionList. +export default function AdoptionView() { + const { t } = useTranslation() + const list = useAdoptionList() + const { state, filters } = list + const { confirm, confirmDialog } = useConfirmDialog() + const searchRef = useRef(null) + const [addAuthor, setAddAuthor] = useState(null) + const [nudgeScan, setNudgeScan] = useState(false) + + // "/" jumps to the search from anywhere in the view but a text field. + useEffect(() => { + const onKey = (e: KeyboardEvent) => { + const el = e.target as HTMLElement | null + if (e.key !== '/' || e.defaultPrevented || el?.closest('input, textarea, select, [contenteditable="true"]')) return + e.preventDefault() + searchRef.current?.focus() + } + document.addEventListener('keydown', onKey) + return () => document.removeEventListener('keydown', onKey) + }, []) + + const scan = () => { + setNudgeScan(false) + void list.startScan() + } + + const ignoreFolder = async (folder: string) => { + if (!await confirm({ + title: t('adoption.ignoreFolder.title', { folder, defaultValue: 'Ignore every book in {{folder}}?' }), + body: t('adoption.ignoreFolder.body', 'They leave this list and later scans keep them out. You can unignore any of them from the Ignored list.'), + confirmLabel: t('adoption.ignoreFolder.confirm', 'Ignore folder'), + })) return + await list.ignoreFolder(folder) + } + + const folders = filters.state === 'pending' ? state.facets?.folders ?? [] : [] + const filtered = filters.search !== '' || filters.reason !== '' || filters.format !== '' || filters.authorFolder !== '' + const empty = state.loaded && !state.loading && state.items.length === 0 && !state.loadError + + const emptyState = () => { + const box = (title: string, body: string, action?: React.ReactNode) => ( +
+

{title}

+

{body}

+ {action &&
{action}
} +
+ ) + if (filtered) { + return box( + t('adoption.empty.filteredTitle', 'No books match these filters'), + t('adoption.empty.filteredBody', 'Try another search, or clear the filters to see every book.'), + , + ) + } + if (filters.state === 'ignored') return box(t('adoption.empty.ignoredTitle', 'Nothing is ignored'), t('adoption.empty.ignoredBody', 'Books you ignore land here, and you can bring any of them back.')) + if (filters.state === 'adopted') return box(t('adoption.empty.adoptedTitle', 'Nothing adopted yet'), t('adoption.empty.adoptedBody', 'Books you adopt stay here for 30 days, with Undo.')) + if (!state.scan?.ran) { + return box( + t('adoption.empty.neverTitle', 'Your library has not been scanned yet'), + t('adoption.empty.neverBody', 'A scan looks through your library folders and lists every book it cannot match to your library, so you can decide what each one is. Files are never moved.'), + , + ) + } + return box( + t('adoption.empty.doneTitle', 'Every book in your library is matched'), + t('adoption.empty.doneBody', 'Nothing needs a decision. New files show up here after the next scan.'), + ) + } + + return ( +
+ {confirmDialog} + + +
0 ? 'lg:grid lg:grid-cols-[15rem_minmax(0,1fr)] lg:gap-6' : ''}> + list.setFilter({ authorFolder: folder })} + onAddAuthor={name => setAddAuthor(name)} + onIgnoreFolder={folder => void ignoreFolder(folder)} + /> +
+ + + {state.loadError && ( +

{state.loadError}

+ )} + {!state.loaded ? ( +
{t('common.loading', 'Loading...')}
+ ) : empty ? ( + emptyState() + ) : ( +
+ searchRef.current?.focus()} onAddAuthor={setAddAuthor} /> +
+ )} + +
+
+ + {addAuthor !== null && ( + setAddAuthor(null)} + onAdded={() => { setAddAuthor(null); setNudgeScan(true) }} + /> + )} +
+ ) +} diff --git a/web/src/pages/import/FolderImportRow.tsx b/web/src/pages/import/FolderImportRow.tsx new file mode 100644 index 000000000..89b446c16 --- /dev/null +++ b/web/src/pages/import/FolderImportRow.tsx @@ -0,0 +1,161 @@ +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { api } from '../../api/client' +import type { BatchImportResult, Book, ScanItem } from '../../api/client' +import { btn, btnSize } from '../../components/buttons' +import BookPicker from '../../components/import/BookPicker' +import CatalogueAdder from '../../components/import/CatalogueAdder' +import { groupHeading, type RowState } from './folderImport' + +function bookLabel(b: Book): string { + return b.author?.authorName ? `${b.title} · ${b.author.authorName}` : b.title +} + +interface ImportRowProps { + item: ScanItem + row: RowState | undefined + result: BatchImportResult | undefined + importing: boolean + onPick: (b: Book) => void + onToggle: () => void + onFormat: (f: string) => void + onImport: () => void +} + +export default function FolderImportRow({ item, row, result, importing, onPick, onToggle, onFormat, onImport }: ImportRowProps) { + const { t } = useTranslation() + const [overriding, setOverriding] = useState(false) + const chosen = row?.chosen ?? null + const accepted = result?.accepted + + const badgeCls: Record = { + confident: 'bg-emerald-100 dark:bg-emerald-950 text-emerald-700 dark:text-emerald-400', + ambiguous: 'bg-amber-100 dark:bg-amber-950 text-amber-700 dark:text-amber-400', + none: 'bg-slate-200 dark:bg-zinc-800 text-slate-600 dark:text-zinc-400', + } + + return ( +
+
+ +
+
+ {item.name} + + {t(`manualImport.group.${item.match}`, groupHeading(item.match))} + + + {item.detectedFormat} + +
+ {/* Full source path so the user can tell which file each row refers to; + the basename alone is ambiguous across folders (#1435). */} +
{item.path}
+
+ {t('manualImport.parsed', { title: item.parsedTitle || '?', author: item.parsedAuthor || '?', defaultValue: `parsed: ${item.parsedTitle || '?'} / ${item.parsedAuthor || '?'}` })} +
+ + {/* Resolution area */} +
+ {chosen && !overriding && ( +
+ → {bookLabel(chosen)} + {!accepted && ( + + )} +
+ )} + + {/* Ambiguous candidate picker */} + {item.match === 'ambiguous' && !chosen && item.candidates && item.candidates.length > 0 && ( +
+ {t('manualImport.pickCandidate', 'Pick the correct book')} + {item.candidates.map(c => ( + + ))} +
+ )} + + {/* None / override: search the existing catalogue, or add the book + from provider metadata when the library has nothing to bind to. */} + {((item.match === 'none' && !chosen) || overriding) && ( + <> + { onPick(b); setOverriding(false) }} + onCancel={overriding ? () => setOverriding(false) : undefined} + /> + {/* searchOnAdd is deliberately false: the file is already on + disk, so an indexer search would grab a second copy. */} + { + const created = await api.addBook({ + foreignBookId: b.foreignBookId, + // May be empty (a Google Books or DNB result carries a + // name but no author id); the backend resolves it by name. + foreignAuthorId: b.author?.foreignAuthorId ?? '', + authorName: b.author?.authorName ?? '', + searchOnAdd: false, + }) + onPick(created) + setOverriding(false) + }} + /> + + )} +
+
+ +
+ + +
+
+ + {result && ( +

+ {accepted + ? t('manualImport.queued', 'Queued — importing in the background; watch the Queue for progress.') + : t('manualImport.failed', { error: result.error || 'failed', defaultValue: `Failed: ${result.error || 'failed'}` })} +

+ )} +
+ ) +} diff --git a/web/src/pages/import/FolderImportView.test.tsx b/web/src/pages/import/FolderImportView.test.tsx index 050f7e30e..75dce2871 100644 --- a/web/src/pages/import/FolderImportView.test.tsx +++ b/web/src/pages/import/FolderImportView.test.tsx @@ -17,7 +17,7 @@ vi.mock('react-i18next', () => ({ }), })) -vi.mock('../api/client', () => ({ +vi.mock('../../api/client', () => ({ api: { scanFolder: vi.fn(), batchImport: vi.fn(), @@ -30,9 +30,9 @@ vi.mock('../api/client', () => ({ }, })) -import { api } from '../api/client' -import type { Book, FolderScanResponse } from '../api/client' -import ManualImportPage from './ManualImportPage' +import { api } from '../../api/client' +import type { Book, FolderScanResponse } from '../../api/client' +import FolderImportView from './FolderImportView' const mockScan = api.scanFolder as ReturnType const mockBatch = api.batchImport as ReturnType @@ -68,7 +68,7 @@ function scanResult(): FolderScanResponse { async function scan() { mockScan.mockResolvedValue(scanResult()) - render() + render() fireEvent.change(screen.getByPlaceholderText('manualImport.pathPlaceholder'), { target: { value: '/dl' } }) fireEvent.click(screen.getByText('manualImport.scan')) await waitFor(() => expect(screen.getByText('Confident Book')).toBeInTheDocument()) @@ -79,7 +79,7 @@ function rowImportButtons() { return screen.getAllByText('manualImport.import') as HTMLButtonElement[] } -describe('ManualImportPage', () => { +describe('FolderImportView', () => { beforeEach(() => vi.clearAllMocks()) it('renders scan results grouped by match status', async () => { diff --git a/web/src/pages/import/FolderImportView.tsx b/web/src/pages/import/FolderImportView.tsx new file mode 100644 index 000000000..472284017 --- /dev/null +++ b/web/src/pages/import/FolderImportView.tsx @@ -0,0 +1,233 @@ +import { useMemo, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { api } from '../../api/client' +import type { BatchImportItem, BatchImportResult, Book, ScanItem } from '../../api/client' +import { btn, btnSize } from '../../components/buttons' +import FolderImportRow from './FolderImportRow' +import { groupHeading, type RowState } from './folderImport' + +// FolderImportView is the Import page's "From a folder" view, the manual +// import wizard (#1236). It scans a folder with +// the recursive bulk-import scan (#1434), groups the discovered units by match +// confidence, and lets the user resolve each one before importing: +// - confident: a preselected catalogue match, deselectable and overridable; +// - ambiguous: a candidate picker over the returned catalogue candidates; +// - none: a search box to bind the unit to an EXISTING catalogue book, plus +// a metadata search that creates the book (and its author) when the library +// has no row to bind to (#1719). +// Resolved units are imported (per-unit or in bulk) through the batch endpoint, +// which reports per-item success/failure. + +const MATCH_ORDER: ScanItem['match'][] = ['confident', 'ambiguous', 'none'] + +export default function FolderImportView() { + const { t } = useTranslation() + const [path, setPath] = useState('') + const [scanning, setScanning] = useState(false) + const [items, setItems] = useState(null) + const [truncated, setTruncated] = useState(false) + const [rows, setRows] = useState>({}) + const [results, setResults] = useState>({}) + const [importingPaths, setImportingPaths] = useState>(() => new Set()) + const [scanError, setScanError] = useState('') + const [importError, setImportError] = useState('') + + const handleScan = async () => { + const p = path.trim() + if (!p) return + setScanning(true) + setScanError('') + setImportError('') + setItems(null) + setResults({}) + try { + const r = await api.scanFolder(p) + setItems(r.items) + setTruncated(r.truncated) + const init: Record = {} + for (const it of r.items) { + const chosen = it.match === 'confident' && it.book ? it.book : null + init[it.path] = { chosen, format: it.detectedFormat || '', selected: Boolean(chosen) } + } + setRows(init) + } catch (e) { + setScanError(e instanceof Error ? e.message : 'Scan failed') + } finally { + setScanning(false) + } + } + + const patchRow = (unitPath: string, patch: Partial) => + setRows(prev => ({ ...prev, [unitPath]: { ...prev[unitPath], ...patch } })) + + const pickBook = (unitPath: string, book: Book) => + patchRow(unitPath, { chosen: book, selected: true }) + + const toggleSelected = (unitPath: string) => + setRows(prev => { + const r = prev[unitPath] + if (!r || !r.chosen) return prev + return { ...prev, [unitPath]: { ...r, selected: !r.selected } } + }) + + // A path is submittable when it has a chosen book, is selected, and hasn't + // already been accepted by a prior import. A `none` unit the user never + // resolved has chosen=null, so it is never submitted. + const submittablePaths = useMemo(() => { + if (!items) return [] + return items + .map(it => it.path) + .filter(p => { + const r = rows[p] + return Boolean(r?.chosen) && r.selected && !results[p]?.accepted + }) + }, [items, rows, results]) + + const submit = async (paths: string[]) => { + const batch: BatchImportItem[] = [] + for (const p of paths) { + const r = rows[p] + if (!r?.chosen || !r.selected) continue + if (results[p]?.accepted) continue + batch.push({ path: p, bookId: r.chosen.id, format: r.format || undefined }) + } + if (batch.length === 0) return + setImportError('') + setImportingPaths(prev => { const n = new Set(prev); batch.forEach(b => n.add(b.path)); return n }) + try { + const res = await api.batchImport(batch) + setResults(prev => { + const n = { ...prev } + for (const r of res.results) n[r.path] = r + return n + }) + } catch (e) { + setImportError(e instanceof Error ? e.message : 'Import failed') + } finally { + setImportingPaths(prev => { const n = new Set(prev); batch.forEach(b => n.delete(b.path)); return n }) + } + } + + const allSelected = submittablePaths.length > 0 && + submittablePaths.every(p => rows[p]?.selected) + + const toggleSelectAll = () => { + if (!items) return + // Resolved, not-yet-imported units only — a `none` unit with no book can't + // be selected. + const resolvable = items + .map(it => it.path) + .filter(p => rows[p]?.chosen && !results[p]?.accepted) + const turnOn = !resolvable.every(p => rows[p]?.selected) + setRows(prev => { + const n = { ...prev } + for (const p of resolvable) n[p] = { ...n[p], selected: turnOn } + return n + }) + } + + const groups = useMemo(() => { + const g: Record = { confident: [], ambiguous: [], none: [] } + for (const it of items ?? []) g[it.match].push(it) + return g + }, [items]) + + return ( +
+

+ {t('manualImport.description', 'Scan a folder of files already on disk, match each book to your library, and import them. A file with no match can be added from a metadata search.')} +

+ +
+ setPath(e.target.value)} + onKeyDown={e => { if (e.key === 'Enter') handleScan() }} + placeholder={t('manualImport.pathPlaceholder', '/downloads/books')} + aria-label={t('manualImport.pathLabel', 'Folder to scan')} + className="flex-1 px-3 py-2 rounded-md border border-slate-300 dark:border-zinc-700 bg-white dark:bg-zinc-950 text-sm" + /> + +
+ + {scanError && ( +

{scanError}

+ )} + + {items && items.length === 0 && ( +

+ {t('manualImport.empty', 'No book files or folders found here.')} +

+ )} + + {items && items.length > 0 && ( + <> + {truncated && ( +

+ {t('manualImport.truncated', 'Showing the first 1000 units; narrow the folder to see the rest.')} +

+ )} + + {/* Select-all / bulk import bar */} +
+ + +
+ + {importError && ( +

{importError}

+ )} + + {MATCH_ORDER.map(match => { + const group = groups[match] + if (group.length === 0) return null + return ( +
+

+ {t(`manualImport.group.${match}`, groupHeading(match))} ({group.length}) +

+
+ {group.map(it => ( + pickBook(it.path, b)} + onToggle={() => toggleSelected(it.path)} + onFormat={f => patchRow(it.path, { format: f })} + onImport={() => submit([it.path])} + /> + ))} +
+
+ ) + })} + + )} +
+ ) +} + diff --git a/web/src/pages/import/ImportPage.test.tsx b/web/src/pages/import/ImportPage.test.tsx new file mode 100644 index 000000000..45e7b7a6b --- /dev/null +++ b/web/src/pages/import/ImportPage.test.tsx @@ -0,0 +1,51 @@ +import { fireEvent, render, screen } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' +import { MemoryRouter, useLocation } from 'react-router' +import ImportPage from './ImportPage' + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ t: (key: string, fallback?: unknown) => (typeof fallback === 'string' ? fallback : key) }), +})) +vi.mock('../../auth/AuthContext', () => ({ useAuth: () => ({ isAdmin: true }) })) +vi.mock('./AdoptionView', () => ({ default: () =>
})) +vi.mock('./FolderImportView', () => ({ default: () =>
})) + +function Location() { + const loc = useLocation() + return {loc.pathname + loc.search} +} + +function renderAt(url: string) { + render( + + + + , + ) +} + +describe('ImportPage', () => { + it('opens on "In your library" by default', () => { + renderAt('/import') + expect(screen.getByRole('tab', { name: 'In your library' })).toHaveAttribute('aria-selected', 'true') + expect(screen.getByTestId('adoption-view')).toBeInTheDocument() + }) + + it('addresses the folder view as ?view=folder, which ImportHints links to', () => { + renderAt('/import?view=folder') + expect(screen.getByRole('tab', { name: 'From a folder' })).toHaveAttribute('aria-selected', 'true') + expect(screen.getByRole('tabpanel')).toContainElement(screen.getByTestId('folder-view')) + }) + + it('switches views by click and by arrow key', () => { + renderAt('/import') + fireEvent.click(screen.getByRole('tab', { name: 'From a folder' })) + expect(screen.getByTestId('location')).toHaveTextContent('/import?view=folder') + + const folderTab = screen.getByRole('tab', { name: 'From a folder' }) + folderTab.focus() + fireEvent.keyDown(folderTab, { key: 'ArrowRight' }) + expect(screen.getByTestId('location')).toHaveTextContent(/^\/import$/) + expect(screen.getByRole('tab', { name: 'In your library' })).toHaveFocus() + }) +}) diff --git a/web/src/pages/import/ImportPage.tsx b/web/src/pages/import/ImportPage.tsx new file mode 100644 index 000000000..837c37e19 --- /dev/null +++ b/web/src/pages/import/ImportPage.tsx @@ -0,0 +1,97 @@ +import { useEffect, useRef, type KeyboardEvent } from 'react' +import { useSearchParams } from 'react-router' +import { useTranslation } from 'react-i18next' +import { useAuth } from '../../auth/AuthContext' +import AdoptionView from './AdoptionView' +import FolderImportView from './FolderImportView' + +type View = 'library' | 'folder' + +const VIEWS: View[] = ['library', 'folder'] + +// ImportPage is where files already on disk come into Bindery, two ways: +// - "In your library" (default): the books a library scan found but could +// not match, adopted in place (AdoptionView); +// - "From a folder" (?view=folder): pick a folder, match and import it, +// moving files into the library (FolderImportView). +// Both used to live apart (a read only table in Settings, a folder scan in +// Settings > Import and this page); this is now the one place. +export default function ImportPage() { + const { t } = useTranslation() + const { isAdmin } = useAuth() + const [params, setParams] = useSearchParams() + const view: View = params.get('view') === 'folder' ? 'folder' : 'library' + const tabRefs = useRef>({ library: null, folder: null }) + + useEffect(() => { + document.title = `${t('importPage.title', 'Import')} · Bindery` + return () => { document.title = 'Bindery' } + }, [t]) + + const select = (next: View) => { + const p = new URLSearchParams(params) + if (next === 'folder') p.set('view', 'folder') + else p.delete('view') + setParams(p, { replace: true }) + } + + // Tabs follow the WAI-ARIA pattern: arrows move between them. + const onTabKey = (e: KeyboardEvent) => { + if (e.key !== 'ArrowRight' && e.key !== 'ArrowLeft') return + e.preventDefault() + const next = VIEWS[(VIEWS.indexOf(view) + (e.key === 'ArrowRight' ? 1 : VIEWS.length - 1)) % VIEWS.length] + select(next) + tabRefs.current[next]?.focus() + } + + const tabCls = (active: boolean) => + `px-3 py-1.5 rounded-md text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-emerald-500 ${ + active + ? 'bg-white dark:bg-zinc-800 text-slate-900 dark:text-white shadow-sm' + : 'text-slate-600 dark:text-zinc-400 hover:text-slate-900 dark:hover:text-white' + }` + + return ( +
+
+
+

{t('importPage.title', 'Import')}

+

+ {t('importPage.subtitle', 'Bring books you already have into Bindery.')} +

+
+
+ {VIEWS.map(v => ( + + ))} +
+
+ +
+ {view === 'folder' ? ( + + ) : isAdmin ? ( + + ) : ( +

+ {t('adoption.adminOnly', 'Only an admin can decide what the files in the library are.')} +

+ )} +
+
+ ) +} diff --git a/web/src/pages/import/adoptionHint.test.ts b/web/src/pages/import/adoptionHint.test.ts new file mode 100644 index 000000000..e9f88518c --- /dev/null +++ b/web/src/pages/import/adoptionHint.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' +import type { TFunction } from 'i18next' +import en from '../../i18n/locales/en.json' +import type { AdoptionItem } from '../../api/client' +import { adoptionHint, scorePercent } from './adoptionHint' + +const lookup = (key: string): unknown => + key.split('.').reduce((n, p) => (n && typeof n === 'object' ? (n as Record)[p] : undefined), en) + +const t = ((key: string, options?: string | Record) => { + const opts = typeof options === 'object' ? options : {} + const value = lookup(key) + return (typeof value === 'string' ? value : key).replace(/\{\{(\w+)\}\}/g, (_, k) => String(opts[k] ?? '')) +}) as unknown as TFunction + +function item(overrides: Partial): AdoptionItem { + return { + id: 1, kind: 'file', format: 'ebook', fileCount: 1, sizeBytes: 1, relPath: 'A/B.epub', rootPath: '/books', + authorFolder: 'A', parsedTitle: 'B', parsedAuthor: 'Becky Chambers', reason: 'no_title_match', candidates: [], + topScore: 0, state: 'pending', bookCreated: false, authorCreated: false, members: [], firstSeenAt: '', + ...overrides, + } +} + +// The row's sentence is written from facts and ends in an action; the raw +// reason code only ever appears in the tooltip (#2033). +describe('adoptionHint', () => { + const codes = ['author_not_in_library', 'no_candidate_books', 'no_title_match', 'no_title_parsed'] as const + + it.each(codes)('never puts the %s code in the sentence, and ends in an action', code => { + const hint = adoptionHint(item({ reason: code }), t) + expect(hint.sentence).not.toContain(code) + expect(hint.sentence).toMatch(/(Add the author, then scan again|Choose the book[^.]*|search metadata)\.$/) + expect(hint.tooltip).toContain(code) + }) + + it('names the author the scan read when the author is missing', () => { + expect(adoptionHint(item({ reason: 'author_not_in_library' }), t).sentence) + .toBe('Becky Chambers is not in your library yet. Add the author, then scan again.') + }) + + it('leads with the suggestion when there is one', () => { + const book = { id: 9, title: 'The Martian', authorId: 1, authorName: 'Andy Weir', status: 'wanted', mediaType: 'ebook', monitored: true } + expect(adoptionHint(item({ candidates: [{ book, score: 0.816 }] }), t).sentence) + .toBe('Closest match is The Martian by Andy Weir (82%). Confirm it or choose another book.') + }) + + it('clamps scores to a percentage', () => { + expect([scorePercent(-1), scorePercent(0.6), scorePercent(1.4)]).toEqual([0, 60, 100]) + }) +}) diff --git a/web/src/pages/import/adoptionHint.ts b/web/src/pages/import/adoptionHint.ts new file mode 100644 index 000000000..38f532b8c --- /dev/null +++ b/web/src/pages/import/adoptionHint.ts @@ -0,0 +1,89 @@ +import type { TFunction } from 'i18next' +import type { AdoptionItem } from '../../api/client' + +// The one sentence a row says about itself. It is written from facts the scan +// recorded (the parsed author, the closest title) and ends in the thing to do +// next. The scanner's reason code is only ever a tooltip: the codes are exact +// but read as blame, and "no title match" was what sent #2033's reporter off +// renaming files that were fine. + +export interface AdoptionHintText { + sentence: string + tooltip: string +} + +// A score is a 0 to 1 Jaro-Winkler similarity; people read a percentage. +export function scorePercent(score: number): number { + return Math.round(Math.max(0, Math.min(1, score)) * 100) +} + +export function adoptionHint(item: AdoptionItem, t: TFunction): AdoptionHintText { + const tooltip = item.reason + ? t('adoption.hint.reasonTooltip', { code: item.reason, defaultValue: 'Scan reason: {{code}}' }) + : '' + const author = item.parsedAuthor + const top = item.candidates[0] + + if (top) { + return { + sentence: t('adoption.hint.suggestion', { + title: top.book.title, + author: top.book.authorName, + percent: scorePercent(top.score), + defaultValue: 'Closest match is {{title}} by {{author}} ({{percent}}%). Confirm it or choose another book.', + }), + tooltip, + } + } + switch (item.reason) { + case 'author_not_in_library': + return { + sentence: author + ? t('adoption.hint.authorMissing', { author, defaultValue: '{{author}} is not in your library yet. Add the author, then scan again.' }) + : t('adoption.hint.authorUnreadable', 'No author could be read from these files. Choose the book they are.'), + tooltip, + } + case 'no_candidate_books': + return { + sentence: t('adoption.hint.noCandidates', { + author: author || item.authorFolder, + defaultValue: '{{author}} has no book in your library waiting for these files. Choose the book or search metadata.', + }), + tooltip, + } + case 'no_title_parsed': + return { + sentence: t('adoption.hint.noTitle', 'No title could be read from the file names or tags. Choose the book they are.'), + tooltip, + } + default: + return { + sentence: author + ? t('adoption.hint.noTitleMatch', { author, defaultValue: 'No book by {{author}} has a title close to this one. Choose the book it is.' }) + : t('adoption.hint.noTitleMatchNoAuthor', 'No book in your library has a title close to this one. Choose the book it is.'), + tooltip, + } + } +} + +// The name a row goes by: the title the scan read, else the last part of its +// path. +export function unitDisplayName(item: AdoptionItem): string { + if (item.parsedTitle) return item.parsedTitle + const parts = item.relPath.split('/') + return parts[parts.length - 1] || item.relPath +} + +// "2 hours ago", in the reader's language. +export function relativeTime(iso: string | undefined, locale?: string): string { + if (!iso) return '' + const then = Date.parse(iso) + if (Number.isNaN(then)) return '' + const seconds = Math.round((then - Date.now()) / 1000) + const rtf = new Intl.RelativeTimeFormat(locale, { numeric: 'auto' }) + const steps: [Intl.RelativeTimeFormatUnit, number][] = [['day', 86400], ['hour', 3600], ['minute', 60]] + for (const [unit, size] of steps) { + if (Math.abs(seconds) >= size) return rtf.format(Math.round(seconds / size), unit) + } + return rtf.format(0, 'minute') +} diff --git a/web/src/pages/import/adoptionReducer.ts b/web/src/pages/import/adoptionReducer.ts new file mode 100644 index 000000000..55dda030e --- /dev/null +++ b/web/src/pages/import/adoptionReducer.ts @@ -0,0 +1,112 @@ +import type { + AdoptionBookRef, AdoptionFacets, AdoptionItem, AdoptionListResponse, AdoptionScanStatus, AdoptionSummary, +} from '../../api/client' + +// Optimistic state for the adoption list, as named actions. A decision shows +// at once as a quiet line on its row; the server's answer then confirms it or +// reverts it with the error inline. The line stays until the next fetch, so a +// person can read what happened and undo it without the row jumping away. + +export type Outcome = + // Adopt sent; preview is the book the person picked, shown before the answer. + | { kind: 'adopting'; preview: AdoptionBookRef | null } + | { kind: 'adopted'; item: AdoptionItem } + | { kind: 'ignoring' } + | { kind: 'ignored' } + | { kind: 'undoing' } + // Undone or unignored while viewing the adopted or ignored list: the row + // has gone back to needing a decision. + | { kind: 'restored' } + +export interface AdoptionListState { + items: AdoptionItem[] + total: number + facets: AdoptionFacets | null + summary: AdoptionSummary | null + scan: AdoptionScanStatus | null + loading: boolean + loaded: boolean + loadError: string + outcomes: Record + errors: Record + expandedId: number | null +} + +export const initialAdoptionState: AdoptionListState = { + items: [], total: 0, facets: null, summary: null, scan: null, + loading: true, loaded: false, loadError: '', outcomes: {}, errors: {}, expandedId: null, +} + +export type AdoptionAction = + | { type: 'loadStarted' } + | { type: 'loaded'; response: AdoptionListResponse } + | { type: 'loadFailed'; error: string } + | { type: 'expanded'; id: number } + | { type: 'collapsed' } + | { type: 'adoptRequested'; id: number; preview: AdoptionBookRef | null } + | { type: 'adoptSucceeded'; id: number; item: AdoptionItem } + | { type: 'ignoreRequested'; id: number } + | { type: 'ignoreSucceeded'; id: number } + | { type: 'undoRequested'; id: number } + | { type: 'undoSucceeded'; id: number; item: AdoptionItem; restored: boolean } + | { type: 'requestFailed'; id: number; error: string; revertTo: Outcome | null } + | { type: 'scanStatus'; summary: AdoptionSummary; scan: AdoptionScanStatus } + +function withOutcome(state: AdoptionListState, id: number, outcome: Outcome | null): AdoptionListState { + const outcomes = { ...state.outcomes } + if (outcome) outcomes[id] = outcome + else delete outcomes[id] + const errors = { ...state.errors } + delete errors[id] + return { ...state, outcomes, errors } +} + +export function adoptionReducer(state: AdoptionListState, action: AdoptionAction): AdoptionListState { + switch (action.type) { + case 'loadStarted': + return { ...state, loading: true, loadError: '' } + case 'loaded': { + const { response } = action + return { + ...state, + loading: false, + loaded: true, + items: response.items, + total: response.total, + // Facets come only with a filter change; a page turn keeps the last. + facets: response.facets ?? state.facets, + summary: response.summary, + scan: response.scan, + outcomes: {}, + errors: {}, + expandedId: response.items.some(i => i.id === state.expandedId) ? state.expandedId : null, + } + } + case 'loadFailed': + return { ...state, loading: false, loaded: true, loadError: action.error } + case 'expanded': + return { ...state, expandedId: action.id } + case 'collapsed': + return { ...state, expandedId: null } + case 'adoptRequested': + return { ...withOutcome(state, action.id, { kind: 'adopting', preview: action.preview }), expandedId: null } + case 'adoptSucceeded': + return withOutcome(state, action.id, { kind: 'adopted', item: action.item }) + case 'ignoreRequested': + return { ...withOutcome(state, action.id, { kind: 'ignoring' }), expandedId: null } + case 'ignoreSucceeded': + return withOutcome(state, action.id, { kind: 'ignored' }) + case 'undoRequested': + return withOutcome(state, action.id, { kind: 'undoing' }) + case 'undoSucceeded': { + const next = withOutcome(state, action.id, action.restored ? { kind: 'restored' } : null) + return { ...next, items: state.items.map(i => (i.id === action.id ? action.item : i)) } + } + case 'requestFailed': { + const next = withOutcome(state, action.id, action.revertTo) + return { ...next, errors: { ...next.errors, [action.id]: action.error } } + } + case 'scanStatus': + return { ...state, summary: action.summary, scan: action.scan } + } +} diff --git a/web/src/pages/import/folderImport.ts b/web/src/pages/import/folderImport.ts new file mode 100644 index 000000000..52d3f1392 --- /dev/null +++ b/web/src/pages/import/folderImport.ts @@ -0,0 +1,23 @@ +import type { Book, ScanItem } from '../../api/client' + +// Shared by FolderImportView and FolderImportRow. + +// RowState is the per-unit resolution the user is building, keyed by unit path. +export interface RowState { + // chosen is the catalogue book the unit will import against (a confident + // match, a picked candidate, or a searched existing book). null until resolved. + chosen: Book | null + // format override: '' = auto-detect, or 'ebook' / 'audiobook'. Defaults to the + // scan's detected format. + format: string + // selected marks the unit for the next bulk import. Only meaningful when chosen. + selected: boolean +} + +export function groupHeading(match: ScanItem['match']): string { + switch (match) { + case 'confident': return 'Matched' + case 'ambiguous': return 'Needs a choice' + default: return 'Unmatched' + } +} diff --git a/web/src/pages/import/useAdoptionList.ts b/web/src/pages/import/useAdoptionList.ts new file mode 100644 index 0000000000000000000000000000000000000000..6d82e93c59b0b47b0d9c9d51ed9c94e135959c42 GIT binary patch literal 6741 zcmd5>ZExGi5$YwMuVnzhUNPd1b1t4X%VJKb(f6l+WPN( zW@eXM>LqQ_0&Ni3d|A%U%QMf+tjqPLv8{SmoikUus_u36IHQ~Ii^60rU*4L$%Z&Bc z1^-^Qx;1L23frvJ#2THole01i)0?t?m!{|Vk0<9@RhoKO);?_vHhz~ko3?D~_pMpa z`nx5xitcDPe|x9xgNfe#p=lm(P4Un7V;h&oem3?-Ka{nmUH3lA8kkT+OLq<*49jBM zyuQ6x%DQYzT?ISFo`L1#^^v5yY0Kj2r+Tg1Y-RGRm98I5{zZ+3#Z5=9JUc(9V1>Ou zl&)3Hwsxvr85MR`pG&iyIF;*G&s9-Yt+8&VHspkd880miRG5)83dR~@%1q@ZD;<2~ zR1!paqo+#OIe+LfU8_w~RjIn_eW#kbdLm)g)VZ;0yV9*HTimvtg*f^+1K9A)DTL$R zrDtd6Z;ZsUhIWO{Ob>-?QCK~nDdh+V^Zph48jpcL*VJ>owlE0~bl0vLd)-vnK6+d< z7S4_yIHPU0iXQ-Xk$l`iT3$lPWTIZwM7xZC=LQ#NyI$=KZffDNLU&ag!r?U^Fn&X9_0~qen zk+dO#F_lYuL=cH}Z>T0IGy_g{vJhMf+So7D;AQ8OdisXVc0Bo+2 z(MWb}jbg3=&o!__Z9oq!nMpxVJyT#tyjBL>@WdJ`A`aD95hk8hn=*R@K);#3VIV)6 zCzyaF$O*_`9OA&0sd4~Mto@30^H{I_+7 zTVxgR5M-7)t=P@f$FY!4$F-BFL(b?6skcFZ&?|CPm^h88X?QC1g!CCBa#-NFrUxOQ zDG#|5nz5adOEui~`n3mFnspY1g@`%$l|sKzzkKx^gfpaezxwKVPZ@eN1atRGBx9@V z)}wTxNZ_l*LXZPojP?xtkUT5uVNeYN#} zQK_;}NzEbNL)Fx$mREq*4RWUkfB@`(mEKY*9AjVr6AX0G!J`AZWmIF5BU&v0yaTO{ zgYH_RV3T}9MRBhB3JZF^YC3QqQ7OTb*^ux!L8=Qv`PY9f?L&l*WceO`qzJ{OXh_LV z%uzNMyd9}Lu0fhEvrQ3EH^a6uikkBFjh9hnId zzdH4;V%Wo10TRIw+@Y-osCpax`?{+td5@nIaB&gJ@(sEltcQBRh8``(l6vo=Gr+Q_ zC_^G0rGe$~9^+*K)Whs1oBoYl5L*JrEGU>k$IDTBA?l4%MIr1_1ClqvUx5HmClhIT zB9(BU!3DtU#62rVdl)Xn9W)sAhgSa&2}nkbIZ>kMnLCi;7%upu4RjQTV*czDA{ z99zRv59jgIRzFdSp~d5$0+F#*yf1Zt8NFwV6lP8tJ zSR7iR(9IO1g9gq*5ry-<(CNhEE}zlvkQ~KSHj{Sigw5k_%=dK?2nLGGkbbI>n_fe? z#Y{RnKx41t7#oy|iIu8*-L3D9We*(^hOG462hvD{X6VTE#^F>!29v|@!U{fWkdK(m z)gw4wB;k=K;wrk{MaQQ6Cz?Pm{Ktiby5*CnI~;s`N3U@pi>5wa^Y-i? DjJvMV literal 0 HcmV?d00001 From 92cb5d1290e523d15c7f9cb53b465c9afa5c2e12 Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 02:25:42 -0300 Subject: [PATCH 09/18] feat(web): fold the old unmatched table and folder scan into the Import page Settings > General no longer lists up to 1000 unmatched files with reason hints nobody could act on; it shows how many books need a decision and a link to review them. The bulk folder scan in Settings > Import becomes a link card to /import?view=folder, and its test goes with it. The empty Queue and Wanted hints point at the two Import views. The per file reason and hint strings are removed from en.json and the one stale translation of them from ko.json. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- web/src/api/library.ts | 10 +- web/src/components/ImportHints.test.tsx | 20 ++ web/src/components/ImportHints.tsx | 11 +- web/src/i18n/locales/en.json | 26 +-- web/src/i18n/locales/ko.json | 1 - web/src/pages/QueuePage.test.tsx | 4 +- web/src/pages/SettingsPage.test.tsx | 93 ++------ web/src/pages/WantedPage.test.tsx | 4 +- .../pages/settings/FolderScanSection.test.tsx | 106 --------- web/src/pages/settings/GeneralTab.tsx | 71 ++---- web/src/pages/settings/ImportTab.tsx | 204 ++---------------- 11 files changed, 88 insertions(+), 462 deletions(-) create mode 100644 web/src/components/ImportHints.test.tsx delete mode 100644 web/src/pages/settings/FolderScanSection.test.tsx diff --git a/web/src/api/library.ts b/web/src/api/library.ts index ad2507c1d..619c1aaac 100644 --- a/web/src/api/library.ts +++ b/web/src/api/library.ts @@ -42,10 +42,12 @@ export const libraryApi = { reconciled: number unmatched: number tag_read_failed?: number - // reason is the scanner's per-file diagnostic (#1958): 'author_not_in_library', - // 'no_candidate_books', 'no_title_match' or 'no_title_parsed'. Optional — - // older cached scan results were written before it existed. - unmatched_files?: Array<{ path: string; parsed_title: string; parsed_author: string; reason?: string }> + // Books the scan could not match (library adoption, /library/unmatched) + // and how many of them are ignored. unmatched_files is always empty now + // and kept only so a cached older bundle still parses. + unmatched_units?: number + ignored_units?: number + units_truncated?: boolean // Additive (feat/library-scan-visibility): the resolved roots the scan // walked and an explicit zero-files signal. Optional so older cached scan // results (persisted before these fields existed) still parse. diff --git a/web/src/components/ImportHints.test.tsx b/web/src/components/ImportHints.test.tsx new file mode 100644 index 000000000..118c3e069 --- /dev/null +++ b/web/src/components/ImportHints.test.tsx @@ -0,0 +1,20 @@ +import { render, screen } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' +import { MemoryRouter } from 'react-router' +import ImportHints from './ImportHints' + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ + t: (key: string) => ({ 'importHints.manualImport': 'Import from a folder', 'importHints.scanLibrary': 'Review my library' }[key] ?? key), + }), +})) + +// The empty Queue and Wanted pages point new users at the Import page, which +// now holds both ways in: a folder elsewhere, and the library folder itself. +describe('ImportHints', () => { + it('links to the Import page views, not to Settings tabs', () => { + render() + expect(screen.getByRole('link', { name: 'Import from a folder' })).toHaveAttribute('href', '/import?view=folder') + expect(screen.getByRole('link', { name: 'Review my library' })).toHaveAttribute('href', '/import') + }) +}) diff --git a/web/src/components/ImportHints.tsx b/web/src/components/ImportHints.tsx index 40ad824d0..52ec5078e 100644 --- a/web/src/components/ImportHints.tsx +++ b/web/src/components/ImportHints.tsx @@ -3,10 +3,9 @@ import { useTranslation } from 'react-i18next' import { btn, btnSize } from './buttons' // Discoverability CTA shown on empty Queue/Wanted states (#1184). New users who -// already have files on disk often don't realise Manual Import / Scan Library -// exist, so we point them straight at the existing flows rather than building a -// new import UI. Both targets are Settings sub-tabs (deep-linked via ?tab=…): -// Manual Import lives under Import / Migrate, Scan Library under General. +// already have files on disk often don't realise Bindery can take them in, so +// we point them straight at the Import page: its folder view for files +// elsewhere, its library view for files already in the library folder. export default function ImportHints() { const { t } = useTranslation() return ( @@ -18,10 +17,10 @@ export default function ImportHints() { {t('importHints.body')}

- + {t('importHints.manualImport')} - + {t('importHints.scanLibrary')}
diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index e8a9eacdf..d1935b373 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -644,8 +644,8 @@ "importHints": { "heading": "Already have files on disk?", "body": "Bindery only auto-imports downloads it grabbed itself. Point it at files you already have, or rescan your library folder.", - "manualImport": "Import them", - "scanLibrary": "Scan Library" + "manualImport": "Import from a folder", + "scanLibrary": "Review my library" }, "blocklist": { "title": "Blocklist", @@ -1007,20 +1007,11 @@ "unmatched": "Unmatched:", "alreadyTracked": "Already tracked:", "scannedPaths": "Scanned:", + "unmatchedUnits_one": "{{count}} book needs a decision.", + "unmatchedUnits_other": "{{count}} books need a decision.", + "reviewUnmatched_one": "Review {{count}} unmatched book", + "reviewUnmatched_other": "Review {{count}} unmatched books", "scanNoFilesWarning": "No book files found under {{path}} — check BINDERY_LIBRARY_DIR points at your library.", - "scanAllUnmatchedHint": "Files were found but none matched a book. Unmatched files need their author's book catalogue populated first — refresh the author (or re-run the import so it fetches books) and scan again.", - "scanUnmatchedAuthorUnknown": "Files were found but none matched a book. The parsed author \"{{author}}\" doesn't match any author in your library, so no book could be considered — check the files' embedded tags and folder names against the Parsed Author column below. Refreshing an author won't help here.", - "scanUnmatchedAuthorsUnknown": "Files were found but none matched a book. None of their parsed authors match an author in your library, so no book could be considered — compare the Parsed Author column below with your authors' names. Refreshing an author won't help here.", - "scanReason": { - "author_not_in_library": "Parsed author isn't in your library", - "no_candidate_books": "Author matched, but has no book waiting for a file", - "no_title_match": "No book by that author matched this title", - "no_title_parsed": "No title could be read from this file's name or tags" - }, - "scanUnmatchedPath": "Path", - "scanUnmatchedParsedTitle": "Parsed Title", - "scanUnmatchedParsedAuthor": "Parsed Author", - "scanUnmatchedWhy": "Why", "opds": "OPDS Feed", "opdsFeedUrl": "OPDS Feed URL", "opdsHint": "Use this URL in KOReader, Moon+ Reader, or any OPDS client. Authenticate with your API key as the password (leave the username blank or use any value).", @@ -1159,7 +1150,10 @@ "goodreadsCommitResult": "{{added}} added · {{skipped}} skipped · {{failed}} failed", "goodreadsNoResolved": "No rows could be resolved — nothing to import. Try widening the shelf filter or check the unmatched rows below.", "goodreadsUploadFailed": "Upload failed", - "configureHardcoverToken": "Configure the Hardcover API token in API Keys settings →" + "configureHardcoverToken": "Configure the Hardcover API token in API Keys settings →", + "folderMovedHeading": "Import from a folder", + "folderMovedBody": "Scanning a folder and importing what it matches is on the Import page now, beside the books a library scan could not match.", + "folderMovedLink": "Open Import" }, "oidc": { "heading": "SSO / OIDC Providers", diff --git a/web/src/i18n/locales/ko.json b/web/src/i18n/locales/ko.json index 895f02527..bffb7b366 100644 --- a/web/src/i18n/locales/ko.json +++ b/web/src/i18n/locales/ko.json @@ -593,7 +593,6 @@ "unmatched": "미일치:", "scannedPaths": "스캔 경로:", "scanNoFilesWarning": "{{path}} 아래에서 도서 파일을 찾을 수 없습니다 — BINDERY_LIBRARY_DIR이 라이브러리를 가리키는지 확인하세요.", - "scanAllUnmatchedHint": "파일은 발견되었으나 일치하는 도서가 없습니다. 미일치 파일은 해당 저자의 도서 목록이 먼저 채워져야 합니다. 저자를 새로고침하거나(또는 가져오기를 다시 실행해 도서를 가져온 후) 다시 스캔하세요.", "opds": "OPDS 피드", "opdsFeedUrl": "OPDS 피드 URL", "opdsHint": "KOReader, Moon+ Reader 또는 OPDS 클라이언트에서 이 URL을 사용하세요. API 키를 비밀번호로 입력해 인증하세요(사용자 이름은 비워 두거나 임의 값을 입력하세요).", diff --git a/web/src/pages/QueuePage.test.tsx b/web/src/pages/QueuePage.test.tsx index 8c65d8cf6..2384a0574 100644 --- a/web/src/pages/QueuePage.test.tsx +++ b/web/src/pages/QueuePage.test.tsx @@ -191,9 +191,9 @@ describe('QueuePage', () => { expect(await screen.findByText('Queue is empty')).toBeInTheDocument() const manualImport = screen.getByRole('link', { name: 'Import them' }) - expect(manualImport).toHaveAttribute('href', '/settings?tab=import') + expect(manualImport).toHaveAttribute('href', '/import?view=folder') const scanLibrary = screen.getByRole('link', { name: 'Scan Library' }) - expect(scanLibrary).toHaveAttribute('href', '/settings?tab=general') + expect(scanLibrary).toHaveAttribute('href', '/import') }) it('renders queue statuses, progress, fallback errors, and error prefixes', async () => { diff --git a/web/src/pages/SettingsPage.test.tsx b/web/src/pages/SettingsPage.test.tsx index 391fdcc40..cc8652d22 100644 --- a/web/src/pages/SettingsPage.test.tsx +++ b/web/src/pages/SettingsPage.test.tsx @@ -775,12 +775,16 @@ describe('SettingsPage', () => { expect(screen.getByText('settings.general.scanNoFilesWarning')).toBeInTheDocument() }) - it('hints to populate catalogue when files found but none matched', async () => { + it('counts unmatched books and links to the Import page instead of listing files', async () => { + // The per file table and its reason hints moved to the Import page's + // "In your library" view, where each book can be acted on. Settings keeps + // the count and the way there. vi.mocked(api.libraryScanStatus).mockResolvedValue({ ran_at: new Date().toISOString(), - files_found: 12, + files_found: 212, reconciled: 0, - unmatched: 12, + unmatched: 212, + unmatched_units: 14, library_dir: '/books', scanned_paths: ['/books'], no_files_found: false, @@ -789,91 +793,24 @@ describe('SettingsPage', () => { renderSettings() expect(await screen.findByText('settings.general.lastScan')).toBeInTheDocument() - expect(screen.getByText('settings.general.scanAllUnmatchedHint')).toBeInTheDocument() - // The no-files warning must NOT appear when files were found. + expect(screen.getByRole('link', { name: /settings\.general\.reviewUnmatched/ })).toHaveAttribute('href', '/import') + expect(screen.queryByRole('table')).not.toBeInTheDocument() expect(screen.queryByText('settings.general.scanNoFilesWarning')).not.toBeInTheDocument() }) - it('names the parsed author instead of blaming the catalogue when no author matched', async () => { - // #1958: the catalogue hint sent a user author-refreshing for two weeks - // while their real problem was the file's tags. When every unmatched file - // reports author_not_in_library, the hint must name the parsed author and - // the old advice must not be shown. + it('shows no review link when nothing needs a decision', async () => { vi.mocked(api.libraryScanStatus).mockResolvedValue({ ran_at: new Date().toISOString(), - files_found: 1, - reconciled: 0, - unmatched: 1, - library_dir: '/books', - scanned_paths: ['/books'], - no_files_found: false, - unmatched_files: [{ - path: '/books/Álvaro Enrigue/You Dreamed of Empires/You Dreamed of Empires.m4b', - parsed_title: 'You Dreamed of Empires', - parsed_author: 'Álvaro Enrigue, Natasha Wimmer - translator, Gabriel Porras', - reason: 'author_not_in_library', - }], - }) - - renderSettings() - - expect(await screen.findByText('settings.general.lastScan')).toBeInTheDocument() - expect(screen.getByText('settings.general.scanUnmatchedAuthorUnknown')).toBeInTheDocument() - expect(screen.queryByText('settings.general.scanAllUnmatchedHint')).not.toBeInTheDocument() - // The per-file diagnosis is rendered alongside the parsed values. - expect(screen.getByText('settings.general.scanReason.author_not_in_library')).toBeInTheDocument() - }) - - it('does not generalise the author hint from a truncated unmatched sample', async () => { - // The scanner caps unmatched_files at 1000 entries. every() over that sample - // proves nothing about the other 4000, so the specific "your authors don't - // match" hint must fall back to the generic advice when the list is capped. - vi.mocked(api.libraryScanStatus).mockResolvedValue({ - ran_at: new Date().toISOString(), - files_found: 5000, - reconciled: 0, - unmatched: 5000, - library_dir: '/books', - scanned_paths: ['/books'], - no_files_found: false, - unmatched_files: [{ - path: '/books/Becky Chambers/A Psalm for the Wild-Built.epub', - parsed_title: 'A Psalm for the Wild-Built', - parsed_author: 'Becky Chambers', - reason: 'author_not_in_library', - }], - }) - - renderSettings() - - expect(await screen.findByText('settings.general.lastScan')).toBeInTheDocument() - expect(screen.getByText('settings.general.scanAllUnmatchedHint')).toBeInTheDocument() - expect(screen.queryByText('settings.general.scanUnmatchedAuthorUnknown')).not.toBeInTheDocument() - }) - - it('keeps the catalogue hint when the author matched but has no candidate books', async () => { - // The original advice is right for #875's case, so a no_candidate_books - // diagnosis must still produce it — the fix narrows the message, it doesn't - // replace it. - vi.mocked(api.libraryScanStatus).mockResolvedValue({ - ran_at: new Date().toISOString(), - files_found: 2, - reconciled: 0, - unmatched: 2, - library_dir: '/books', - scanned_paths: ['/books'], - no_files_found: false, - unmatched_files: [ - { path: '/books/Andy Weir/Project Hail Mary.epub', parsed_title: 'Project Hail Mary', parsed_author: 'Andy Weir', reason: 'no_candidate_books' }, - { path: '/books/Andy Weir/The Martian.epub', parsed_title: 'The Martian', parsed_author: 'Andy Weir', reason: 'author_not_in_library' }, - ], + files_found: 3, + reconciled: 3, + unmatched: 0, + unmatched_units: 0, }) renderSettings() expect(await screen.findByText('settings.general.lastScan')).toBeInTheDocument() - expect(screen.getByText('settings.general.scanAllUnmatchedHint')).toBeInTheDocument() - expect(screen.queryByText('settings.general.scanUnmatchedAuthorUnknown')).not.toBeInTheDocument() + expect(screen.queryByRole('link', { name: /settings\.general\.reviewUnmatched/ })).not.toBeInTheDocument() }) it('soft-navigates to Root Folders from the Default library location link', async () => { diff --git a/web/src/pages/WantedPage.test.tsx b/web/src/pages/WantedPage.test.tsx index 3fd57f8d4..a6bbea40d 100644 --- a/web/src/pages/WantedPage.test.tsx +++ b/web/src/pages/WantedPage.test.tsx @@ -199,9 +199,9 @@ describe('WantedPage', () => { expect(await screen.findByText('No wanted books. Add an author to start tracking.')).toBeInTheDocument() const manualImport = screen.getByRole('link', { name: 'Import them' }) - expect(manualImport).toHaveAttribute('href', '/settings?tab=import') + expect(manualImport).toHaveAttribute('href', '/import?view=folder') const scanLibrary = screen.getByRole('link', { name: 'Scan Library' }) - expect(scanLibrary).toHaveAttribute('href', '/settings?tab=general') + expect(scanLibrary).toHaveAttribute('href', '/import') }) it('renders a row with the book title and its author', async () => { diff --git a/web/src/pages/settings/FolderScanSection.test.tsx b/web/src/pages/settings/FolderScanSection.test.tsx deleted file mode 100644 index 363df318d..000000000 --- a/web/src/pages/settings/FolderScanSection.test.tsx +++ /dev/null @@ -1,106 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen, fireEvent, waitFor } from '@testing-library/react' - -// i18n: return the key (with crude interpolation) so assertions are stable. -vi.mock('react-i18next', () => ({ - useTranslation: () => ({ - t: (key: string, options?: unknown) => { - // t(key) and t(key, 'default string') both render the bare key for - // stable assertions; t(key, {opts}) appends the interpolation values. - if (!options || typeof options !== 'object') return key - let out = key - for (const [k, v] of Object.entries(options as Record)) { - if (k === 'defaultValue') continue - out += ` ${k}=${String(v)}` - } - return out - }, - }), -})) - -vi.mock('../../api/client', () => ({ - api: { - scanFolder: vi.fn(), - batchImport: vi.fn(), - }, -})) - -import { api, FolderScanResponse } from '../../api/client' -import { FolderScanSection } from './ImportTab' - -const mockScan = api.scanFolder as ReturnType -const mockBatch = api.batchImport as ReturnType - -function scanResult(): FolderScanResponse { - return { - truncated: false, - items: [ - { - path: '/dl/Confident Book', name: 'Confident Book', match: 'confident', - parsedTitle: 'Confident Book', parsedAuthor: 'A. Writer', detectedFormat: 'ebook', - book: { id: 11, title: 'Confident Book', author: { authorName: 'A. Writer' } } as never, - }, - { - path: '/dl/Maybe Book', name: 'Maybe Book', match: 'ambiguous', - parsedTitle: 'Maybe Book', parsedAuthor: '', detectedFormat: 'ebook', - candidates: [ - { id: 21, title: 'Maybe Book (1)', author: { authorName: 'X' } } as never, - { id: 22, title: 'Maybe Book (2)', author: { authorName: 'Y' } } as never, - ], - }, - { - path: '/dl/Orphan', name: 'Orphan', match: 'none', - parsedTitle: 'Orphan', parsedAuthor: '', detectedFormat: 'audiobook', - }, - ], - } -} - -describe('FolderScanSection', () => { - beforeEach(() => vi.clearAllMocks()) - - async function scan() { - mockScan.mockResolvedValue(scanResult()) - render() - fireEvent.change(screen.getByPlaceholderText('settings.import.bulkPathPlaceholder'), { - target: { value: '/dl' }, - }) - fireEvent.click(screen.getByText('settings.import.bulkScan')) - await waitFor(() => expect(screen.getByText('Confident Book')).toBeInTheDocument()) - } - - it('pre-selects only confident matches and imports them', async () => { - await scan() - expect(mockScan).toHaveBeenCalledWith('/dl') - - const checkboxes = screen.getAllByRole('checkbox') as HTMLInputElement[] - expect(checkboxes[0].checked).toBe(true) // confident - expect(checkboxes[1].checked).toBe(false) // ambiguous, no pick yet - expect(checkboxes[2].disabled).toBe(true) // none, not selectable - - // Import button reflects 1 selected (the confident one). - expect(screen.getByText(/settings\.import\.bulkImport count=1/)).toBeInTheDocument() - - mockBatch.mockResolvedValue({ accepted: 1, failed: 0, results: [{ path: '/dl/Confident Book', accepted: true, downloadId: 5 }] }) - fireEvent.click(screen.getByText(/settings\.import\.bulkImport count=1/)) - - await waitFor(() => expect(mockBatch).toHaveBeenCalledTimes(1)) - expect(mockBatch).toHaveBeenCalledWith([{ path: '/dl/Confident Book', bookId: 11, format: 'ebook' }]) - await waitFor(() => expect(screen.getByText(/bulkSummary accepted=1 failed=0/)).toBeInTheDocument()) - }) - - it('lets an ambiguous match be resolved via the picker and included', async () => { - await scan() - // Pick candidate 22 for the ambiguous row. - fireEvent.change(screen.getByRole('combobox'), { target: { value: '22' } }) - expect(screen.getByText(/settings\.import\.bulkImport count=2/)).toBeInTheDocument() - - mockBatch.mockResolvedValue({ accepted: 2, failed: 0, results: [] }) - fireEvent.click(screen.getByText(/settings\.import\.bulkImport count=2/)) - await waitFor(() => expect(mockBatch).toHaveBeenCalledTimes(1)) - expect(mockBatch).toHaveBeenCalledWith([ - { path: '/dl/Confident Book', bookId: 11, format: 'ebook' }, - { path: '/dl/Maybe Book', bookId: 22, format: 'ebook' }, - ]) - }) -}) diff --git a/web/src/pages/settings/GeneralTab.tsx b/web/src/pages/settings/GeneralTab.tsx index edabf99bc..e67d462be 100644 --- a/web/src/pages/settings/GeneralTab.tsx +++ b/web/src/pages/settings/GeneralTab.tsx @@ -1,7 +1,7 @@ import { FormEvent, useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { useConfirmDialog } from '../../components/useConfirmDialog' -import { api, AuthConfig, AuthStatus, StorageDirStatus, StorageHealth } from '../../api/client' +import { api, AuthConfig, AuthStatus, BINDERY_BASE, StorageDirStatus, StorageHealth } from '../../api/client' import AuthSettings from '../../settings/AuthSettings' import ThemeToggle from '../../components/ThemeToggle' import LanguageSwitcher from '../../components/LanguageSwitcher' @@ -42,9 +42,10 @@ export default function GeneralTab({ onNavigate }: GeneralTabProps = {}) { unmatched: number already_tracked?: number tag_read_failed?: number - // reason is one of the scanner's unmatchedReason* codes (#1958); absent on - // scan results persisted before that field existed. - unmatched_files?: Array<{ path: string; parsed_title: string; parsed_author: string; reason?: string }> + // Books (not files) the scan could not match, and how many are ignored. + // Absent on results persisted before library adoption. + unmatched_units?: number + ignored_units?: number library_dir?: string audiobook_dir?: string scanned_paths?: string[] @@ -560,61 +561,17 @@ export default function GeneralTab({ onNavigate }: GeneralTabProps = {}) { })}

)} - {lastScan.files_found > 0 && lastScan.unmatched > 0 && lastScan.reconciled === 0 && ( -

- {(() => { - // The catalogue advice is only right when the author matched - // but has no book to attach a file to. When every unmatched - // file's parsed author matches NO author in the library, - // refreshing an author cannot help — name the parsed author - // instead, which is the evidence to act on (#1958). - // unmatched_files is a sample capped at 1000 by the scanner; - // every() over it says nothing about the rest, so a run with - // 5000 unmatched files whose first 1000 happen to share a - // reason must not claim the diagnosis holds library-wide. - const files = lastScan.unmatched_files ?? [] - const truncated = files.length < lastScan.unmatched - if (truncated || files.length === 0 || !files.every(f => f.reason === 'author_not_in_library')) { - return t('settings.general.scanAllUnmatchedHint') - } - const authors = Array.from(new Set(files.map(f => f.parsed_author).filter(Boolean))) - return authors.length === 1 - ? t('settings.general.scanUnmatchedAuthorUnknown', { author: authors[0] }) - : t('settings.general.scanUnmatchedAuthorsUnknown') - })()} + {/* The unmatched files used to be listed here, capped at 1000 + per file rows with no way to act on one. They are books to + decide about on the Import page now; this only counts them. */} + {(lastScan.unmatched_units ?? 0) > 0 && ( +

+ {t('settings.general.unmatchedUnits', { count: lastScan.unmatched_units, defaultValue: '{{count}} books need a decision.' })} + + {t('settings.general.reviewUnmatched', { count: lastScan.unmatched_units, defaultValue: 'Review {{count}} unmatched books' })} +

)} - {lastScan.unmatched_files && lastScan.unmatched_files.length > 0 && ( -
- - Unmatched files ({lastScan.unmatched_files.length}{lastScan.unmatched_files.length >= 1000 && lastScan.unmatched > 1000 ? ' of ' + lastScan.unmatched : ''}) - -
- - - - - - - - - - - {lastScan.unmatched_files.map((file, idx) => ( - - - - - {/* Per-file diagnosis from the scanner (#1958). Blank - for results persisted before the field existed. */} - - - ))} - -
{t('settings.general.scanUnmatchedPath')}{t('settings.general.scanUnmatchedParsedTitle')}{t('settings.general.scanUnmatchedParsedAuthor')}{t('settings.general.scanUnmatchedWhy')}
{file.path}{file.parsed_title || '—'}{file.parsed_author || '—'}{file.reason ? t(`settings.general.scanReason.${file.reason}`) : '—'}
-
-
- )}
)}
diff --git a/web/src/pages/settings/ImportTab.tsx b/web/src/pages/settings/ImportTab.tsx index fce364968..eee2c0555 100644 --- a/web/src/pages/settings/ImportTab.tsx +++ b/web/src/pages/settings/ImportTab.tsx @@ -1,6 +1,6 @@ import { useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' -import { api, BatchImportItem, BatchImportResponse, Book, HardcoverList, ImportList, ImportListSyncProgress, ManagedUser, ManualImportLookup, ScanItem } from '../../api/client' +import { api, BINDERY_BASE, Book, HardcoverList, ImportList, ImportListSyncProgress, ManagedUser, ManualImportLookup } from '../../api/client' import { inputCls } from './formStyles' import GoodreadsImportSection from './GoodreadsImportSection' @@ -118,7 +118,7 @@ export default function ImportTab({ onNavigate }: ImportTabProps = {}) { )} - + @@ -263,198 +263,22 @@ function ManualImportSection() { ) } -interface ScanRowState { - include: boolean - bookId: number | null - format: string -} - -// FolderScanSection scans a folder for book units and bulk-imports the selected -// matches in one shot, so a migration backlog doesn't have to be imported one -// path at a time. Matches against the existing catalogue (add the authors/books -// first); unmatched units are listed but cannot be selected. Exported for tests. -export function FolderScanSection() { +// FolderImportLink stands where the bulk folder scan used to be. That scan +// now lives on the Import page, next to the view for files already in the +// library, so there is one place to bring books in rather than two. +function FolderImportLink() { const { t } = useTranslation() - const [path, setPath] = useState('') - const [scanning, setScanning] = useState(false) - const [items, setItems] = useState(null) - const [truncated, setTruncated] = useState(false) - const [rows, setRows] = useState([]) - const [importing, setImporting] = useState(false) - const [summary, setSummary] = useState(null) - const [err, setErr] = useState(null) - - const handleScan = async () => { - if (!path.trim()) return - setScanning(true) - setErr(null) - setItems(null) - setSummary(null) - try { - const r = await api.scanFolder(path.trim()) - setItems(r.items) - setTruncated(r.truncated) - setRows(r.items.map(it => ({ - include: it.match === 'confident', - bookId: it.match === 'confident' && it.book ? it.book.id : null, - format: it.detectedFormat || '', - }))) - } catch (e) { - setErr(e instanceof Error ? e.message : 'Scan failed') - } finally { - setScanning(false) - } - } - - const patchRow = (i: number, patch: Partial) => - setRows(prev => prev.map((r, idx) => (idx === i ? { ...r, ...patch } : r))) - - const selectedCount = rows.filter(r => r.include && r.bookId).length - - const handleImport = async () => { - if (!items) return - const batch: BatchImportItem[] = [] - rows.forEach((r, i) => { - if (r.include && r.bookId) batch.push({ path: items[i].path, bookId: r.bookId, format: r.format || undefined }) - }) - if (batch.length === 0) return - setImporting(true) - setErr(null) - try { - const res = await api.batchImport(batch) - setSummary(res) - } catch (e) { - setErr(e instanceof Error ? e.message : 'Import failed') - } finally { - setImporting(false) - } - } - - const matchBadge = (m: ScanItem['match']) => { - const map: Record = { - confident: 'bg-emerald-100 dark:bg-emerald-950 text-emerald-700 dark:text-emerald-400', - ambiguous: 'bg-amber-100 dark:bg-amber-950 text-amber-700 dark:text-amber-400', - none: 'bg-slate-200 dark:bg-zinc-800 text-slate-500 dark:text-zinc-500', - } - return {t(`settings.import.bulkMatch_${m}`, m)} - } - return (
-

{t('settings.import.bulkHeading', 'Bulk folder import')}

-

- {t('settings.import.bulkDescription', 'Scan a folder (e.g. your download directory) and import every book it can match to your library in one go. Matching is against books already in your library, so add the authors first. Unmatched items are listed but skipped.')} -

- -
- { setPath(e.target.value); setItems(null); setSummary(null) }} - onKeyDown={e => { if (e.key === 'Enter') handleScan() }} - /> - +

{t('settings.import.folderMovedHeading', 'Import from a folder')}

+
+

+ {t('settings.import.folderMovedBody', 'Scanning a folder and importing what it matches is on the Import page now, beside the books a library scan could not match.')} +

+ + {t('settings.import.folderMovedLink', 'Open Import')} +
- - {items && items.length === 0 && ( -

{t('settings.import.bulkEmpty', 'No book files or folders found here.')}

- )} - - {items && items.length > 0 && !summary && ( -
- {truncated && ( -

{t('settings.import.bulkTruncated', 'Showing the first 1000 items; narrow the folder to see the rest.')}

- )} -
- {items.map((it, i) => { - const row = rows[i] - const disabled = it.match === 'none' - return ( -
- patchRow(i, { include: e.target.checked })} - aria-label={t('settings.import.bulkSelect', { name: it.name, defaultValue: `Import ${it.name}` })} - /> -
-
- {it.name} - {matchBadge(it.match)} - {it.detectedFormat} -
- {/* Full source path so the user can tell which file a match - (or the ambiguous picker below) refers to — the basename - alone is ambiguous across folders (#1435). */} -
{it.path}
-
- {t('settings.import.bulkParsed', { title: it.parsedTitle || '?', author: it.parsedAuthor || '?', defaultValue: `parsed: ${it.parsedTitle || '?'} / ${it.parsedAuthor || '?'}` })} -
- {it.match === 'confident' && it.book && ( -
→ {it.book.title}{it.book.author ? ` (${it.book.author.authorName})` : ''}
- )} - {it.match === 'ambiguous' && ( - - )} - {it.match === 'none' && ( -
{t('settings.import.bulkNoMatch', 'No catalogue match; add the book first.')}
- )} -
-
- ) - })} -
- -
- )} - - {summary && ( -
-
{t('settings.import.bulkSummary', { accepted: summary.accepted, failed: summary.failed, defaultValue: `${summary.accepted} queued, ${summary.failed} failed` })}
- {summary.failed > 0 && ( -
- {t('settings.import.bulkShowFailures', { count: summary.failed, defaultValue: `Show ${summary.failed} failures` })} -
    - {summary.results.filter(r => !r.accepted).map(r => ( -
  • {r.path}: {r.error}
    • - ))} -
-
- )} -

{t('settings.import.bulkSummaryNote', 'Imports run in the background; watch the Queue for progress.')}

-
- )} - - {err &&

{err}

}
) } From 4a531701c6388041bc496618dbaf1531b0f55aa2 Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 02:27:02 -0300 Subject: [PATCH 10/18] docs: document library adoption on the Import page A new user guide section, "Adopting files already in your library", the /library/unmatched routes and their answers in API.md, a troubleshooting entry for files a scan leaves unmatched, a README feature line, and the changelog fragment. References to Manual Import, the bulk folder import and the Settings unmatched table point at the Import page views now. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- README.md | 1 + changelog.d/library-adoption.md | 5 ++ docs/API.md | 59 +++++++++++++++++- docs/Troubleshooting-Wiki.md | 27 ++++++++- docs/User-Guide-Wiki.md | 104 ++++++++++++++++++++++++++------ 5 files changed, 175 insertions(+), 21 deletions(-) create mode 100644 changelog.d/library-adoption.md diff --git a/README.md b/README.md index 8e0142849..889879e99 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ Clean-room Go rewrite, modern React UI, MIT-licensed, actively developed. - Dual-format books — each title holds an ebook *and* an audiobook in independent slots, with separate search, grab, and import pipelines, and the audiobook side moves multi-part `.m4b` / `.mp3` folders as one unit. - Series support with position tracking and search by series title, edition tracking (format / ISBN / publisher / page count), Calendar view of upcoming releases, and multiple library roots. - Library scan with four-tier matching: ASIN → title + author → series name + position → fuzzy title. Honours librarian sort-suffix form (`Title, The`) and series-annotated filenames (`[Mistborn, Book 1]`). +- **Library adoption**: whatever the scan could not match waits on the Import page as one row per book (an audiobook folder or disc set is one row), with suggested matches, a one click Confirm, a folder rail for "add this author" decisions, and Undo. Files are registered where they are, never moved. - Author aliases (`RR Haywood` / `R.R. Haywood` / `R R Haywood` merge into one canonical row), and metadata re-bind to correct a wrong match without delete-and-re-add. On a book’s **File → Re-bind** dialog, search configured metadata providers and select an OpenLibrary work or Hardcover book, or enter an exact provider ID and press Enter. Results already in the library are marked and link to the existing book; records belonging to another library book cannot be selected. Each result’s **Links** menu opens its upstream page when available; OpenLibrary editions can be viewed but require a work ID for re-binding. Re-binding updates metadata and series without moving files; author mismatches require confirmation. - Explicit author-catalogue reconciliation with a selectable preview: remove chosen stale metadata-only Wanted rows after changing provider or metadata profile while always protecting imported books and every row with a tracked file. diff --git a/changelog.d/library-adoption.md b/changelog.d/library-adoption.md new file mode 100644 index 000000000..2d3f4a3d2 --- /dev/null +++ b/changelog.d/library-adoption.md @@ -0,0 +1,5 @@ +### Added +- **Adopt the books a library scan could not match** (#2547): they now wait on the Import page under **In your library**, which is where `/import` opens. Each row is a book rather than a file, so a 193 track audiobook is one row, a disc set is one row named after its folder, and `Dune.epub` beside `Dune.mobi` is one row. A row with a close match in your library is one click to confirm. Any row opens in place to pick a book, search your library, or search metadata, and metadata providers are only asked when you press Search. A rail of author folders turns "this whole author is missing" into one Add author. Adopting registers the files where they are: nothing is moved and nothing is searched for, a book added from metadata comes in unmonitored in the format you adopted, and Undo reverses it exactly. Ignored books stay hidden across scans until you unignore them. Admins see how many books need a decision on the Import nav entry. + +### Changed +- **The unmatched files table in Settings and the bulk folder scan in Settings > Import moved to the Import page.** Settings now shows how many books need a decision with a link to review them, and the folder scan is **Import > From a folder**. The old list stopped at 1000 files; the new one has no such cap for ordinary libraries (it lists up to 20,000 books per scan and says when there are more). The `library.lastScan` summary keeps its counters, adds `unmatched_units`, `ignored_units` and `units_truncated`, and returns `unmatched_files` as an empty list for one release. diff --git a/docs/API.md b/docs/API.md index 515c3c7f1..b671c6160 100644 --- a/docs/API.md +++ b/docs/API.md @@ -374,6 +374,13 @@ POST /api/v1/backup/{filename}/restore stage a backup for the next re GET /api/v1/system/status version, uptime, build info POST /api/v1/library/scan start a library scan in the background (202) GET /api/v1/library/scan/status summary of the last library scan, paths included (admin) +GET /api/v1/library/unmatched books the scan could not match, one row per book (admin) +GET /api/v1/library/unmatched/summary pending, ignored and adopted counts plus scan status (admin) +POST /api/v1/library/unmatched/{id}/adopt register the row's files in place against a book (admin) +POST /api/v1/library/unmatched/{id}/undo reverse an adoption exactly (admin) +POST /api/v1/library/unmatched/{id}/ignore set a pending row aside (admin) +POST /api/v1/library/unmatched/{id}/unignore return an ignored row to pending (admin) +POST /api/v1/library/unmatched/ignore ignore pending rows by {"ids":[..]} or {"authorFolder":"..."} (admin) PUT /api/v1/system/loglevel runtime log-level switch (debug/info/warn/error) GET /api/v1/images?url= proxied + cached cover image (30-day TTL) ``` @@ -382,7 +389,57 @@ GET /api/v1/images?url= proxied + cached cover image ( scan: the counts, the library roots it walked and the path of every unmatched file. That is server filesystem layout, so the route is admin only in the same way as `/system/storage`, and a non admin gets `403` (#2361). A `404` means no -scan has run yet. +scan has run yet. The unmatched files themselves moved to +`/library/unmatched`: the summary now carries `unmatched_units`, +`ignored_units` and `units_truncated`, and `unmatched_files` is always an +empty list, kept for one release so an older cached web bundle still parses. + +#### Library adoption + +`GET /api/v1/library/unmatched` lists the books a library scan could not +match. A row is a book: an audiobook folder, a disc set, or same named ebook +files are one row. Query parameters, all optional: + +| Parameter | Values | +|---|---| +| `state` | `pending` (default), `ignored`, `adopted` | +| `reason` | `author_not_in_library`, `no_candidate_books`, `no_title_match`, `no_title_parsed` | +| `authorFolder` | first folder under the library root | +| `format` | `ebook`, `audiobook` | +| `search` | words matched against title, author and path | +| `sort`, `dir` | `score` (default), `title`, `folder`, `files`, `size`, `seen`; `asc` or `desc` | +| `limit`, `offset` | page, at most 250 rows | +| `facets` | `1` adds grouped counts by reason, format and top author folders | + +The response is `{items, total, facets?, summary, scan}`. Each item carries +`id`, `kind`, `format`, `fileCount`, `sizeBytes`, `relPath`, `rootPath`, +`authorFolder`, `parsedTitle`, `parsedAuthor`, `reason`, up to three +`candidates` (`{book, score}`, a title similarity from 0 to 1), `state`, the +adopted `book` if any, `bookCreated`, `authorCreated` and the first 20 +`members` file names. No provider is called to build it. + +`POST /api/v1/library/unmatched/{id}/adopt` takes either `{"bookId": 12}` for +a book already in the library or `{"foreignBookId": "...", "foreignAuthorId": +"...", "authorName": "..."}` for a metadata result, plus an optional +`"format": "ebook" | "audiobook"`. No field is a path. The files are +registered in place (an audiobook folder as its folder), nothing is moved and +no search starts. A book added from metadata is created unmonitored with its +media type set to the adopted format. Answers: + +| Status | Meaning | +|---|---| +| `200` | the updated row | +| `400` | neither or both of `bookId` and `foreignBookId`, or a bad format | +| `404` | no such row, or the book is gone | +| `409` | the row is not pending (the body names its `state`), or a file already belongs to a book | +| `422` | a file is gone, is not a regular file, or resolves outside the library folders | +| `502`, `503` | the metadata provider failed or the primary provider is down | + +`POST .../undo` removes exactly the book file entries the adoption made, and a +book or author it created when nothing else holds them, and returns the row to +pending. Adopt, undo, ignore and unignore each claim the row with a compare and +swap, so a double click or two admins acting at once get one `200` and one +`409`. #### Webhook payload diff --git a/docs/Troubleshooting-Wiki.md b/docs/Troubleshooting-Wiki.md index ebd10467a..34c9fd192 100644 --- a/docs/Troubleshooting-Wiki.md +++ b/docs/Troubleshooting-Wiki.md @@ -313,8 +313,8 @@ co-author or a joint pen name. The log shows lines like: DEBUG calibre import: alias record skipped error="alias \"Isaac Asimov\" already points at author 1125 (refusing to reassign to 105)" name="Isaac Asimov" ``` -The scan then reports those same names under **Unmatched files** with "Parsed -author isn't in your library", because the name only exists as an alias. +The scan then lists those books on **Import → In your library** as by an +author who is not in your library, because the name only exists as an alias. What happened: for a book credited to several people, the import used to record every co-author as an *alias* of the first credited author ([#1684](https://github.com/vavallee/bindery/issues/1684)). @@ -374,6 +374,29 @@ Two things this does not do: If you reshape your library regularly, **Rename files** on the book or author page is the supported way to do it: it moves the file *and* repoints the same tracking row at the new location, so there is never a second row to clean up. +## A scan leaves files unmatched + +Go to **Import → In your library**. Every book the last scan could not match +is listed there, one row per book, with a sentence saying what the scan found +and what to do: add the missing author and scan again, confirm a suggested +book, or choose one. Adopting registers the files where they are and can be +undone. The full walkthrough is in the user guide under [Adopting files +already in your library](User-Guide-Wiki.md#adopting-files-already-in-your-library). + +A few things that look wrong but are not: + +- **A file you expected is not listed.** Symlinks are never listed, and a file + whose folder resolves outside your library folders is skipped. Replace the + link with the file, or add the real folder as a root. +- **The list did not change after the volume came back.** A scan that finds no + files at all changes nothing on the list, so an unmounted library does not + erase your decisions. Scan again once it is mounted. +- **A row you ignored is gone for good.** It is on the **Ignored** list, and + **Unignore** brings it back. + +If you file a bug about a row, include the reason code that shows when you +hover its sentence. + ## Collecting logs for a bug report `Settings → Logs` is the whole log store, so you don't need shell access to the container (rootless images give you nowhere to `cat` a file anyway). diff --git a/docs/User-Guide-Wiki.md b/docs/User-Guide-Wiki.md index f4ec4308c..959e9c237 100644 --- a/docs/User-Guide-Wiki.md +++ b/docs/User-Guide-Wiki.md @@ -41,10 +41,13 @@ an arbitrary release is unreliable. The escape hatches: - **Search Indexers** (the magnifier icon in the header) — free-text search across all your indexers; any result can be grabbed. -- **Manual Import** (`/import`) — point it at a folder of files you already - have and match them to books. A file with no catalogue match gets a metadata - search on its row, which creates the book (and its author, if new) and links - the file to it, so an unmatched file is no longer a dead end. +- **Import** (`/import`): **In your library** lists the books a library scan + could not match, for you to adopt in place ([Adopting files already in your + library](#adopting-files-already-in-your-library)). **From a folder** points + at a folder of files you already have and matches them to books. Either way a + file with no catalogue match gets a metadata search, which creates the book + (and its author, if new) and links the file to it, so an unmatched file is no + longer a dead end. Both still end by attaching a file to a catalogue record. @@ -67,9 +70,10 @@ Almost every "why is Bindery doing that?" question comes down to one of these. **Scan Library** (Settings → General → Library, and automatically every 6 hours) matches files already on disk to books **already in your catalogue**. -It never creates authors or books from files. If you point a fresh install at -a folder of 3,000 epubs and hit Scan, you get 3,000 "unmatched" files and an -empty library — nothing populated the catalogue first. +It never creates authors or books from files on its own. If you point a fresh +install at a folder of 3,000 epubs and hit Scan, you get a long list of +unmatched books on **Import → In your library** and an empty library, because +nothing populated the catalogue first. That list is where you can adopt them. The sequence that works is always: **populate the catalogue, then scan**. See [Bringing in an existing library](#bringing-in-an-existing-library) below. @@ -113,8 +117,9 @@ Consequences: save path Bindery hands qBittorrent. - There is no per-protocol (torrent vs usenet) folder setting because each client already owns its completed path. -- Files you acquired outside Bindery are picked up only via **Manual Import**, - **Bulk folder import**, or a **Library Scan** (rule 1 applies). +- Files you acquired outside Bindery are picked up only via **Import → From a + folder**, or a **Library Scan** and **Import → In your library** for files + already in the library folder (rule 1 applies). ### 4. Naming templates are for output, not input @@ -323,7 +328,7 @@ matches where your metadata lives, then scan: | An Audiobookshelf server | Settings → Audiobookshelf → configure + **Import** ([guide](ABS-Import-Wiki.md)) | | A Goodreads account | Settings → Import → **Goodreads CSV** (export, filter by shelf, preview, commit) | | Just a list of authors | Settings → Import → paste or upload the author list | -| Only folders of files | Use **Manual Import** (`/import`) or Settings → Import → **Bulk folder import**, which match files and create what's missing | +| Only folders of files | Scan the library, then adopt on **Import → In your library**; or use **Import → From a folder** for files outside the library | Then run **Settings → General → Library → Scan Library** to attach your files to the records. Things worth knowing before you judge the results: @@ -373,11 +378,10 @@ to the records. Things worth knowing before you judge the results: - ABS imports that "lose" titles usually didn't: ambiguous matches are parked in the **review queue** (Settings → Audiobookshelf) for you to resolve, and the import summary counts them. -- Unmatched files are listed after the scan, each with the reason it missed: - the parsed author isn't in your library (fix the file's tags or folder name), - the author matched but has no book waiting for a file (populate that author's - catalogue), no title matched, or no title could be read from the file at all - (rename it). Use **Manual Import** to resolve the rest by hand. +- Books the scan could not match wait on **Import → In your library**, one row + per book with a sentence saying why and what to do: add the author, confirm + a suggested book, or choose one. See [Adopting files already in your + library](#adopting-files-already-in-your-library). - **Fix match moves and renames the file.** When a book page shows the wrong file, the **Fix match** button reassigns it to the book you pick. That runs the full import, so the file is moved into the target book's folder and @@ -404,6 +408,67 @@ to the records. Things worth knowing before you judge the results: as evidence you own the book, and a real ebook wins over a supplement-class file when both match (#2240). +## Adopting files already in your library + +A library scan attaches every file it can match with confidence and leaves +the rest for you. Those books wait on **Import → In your library**, the page +`/import` opens on. Each row is one **book**, not one file: a 193 track +audiobook folder is one row, a folder of disc folders (`CD1`, `Disc 2`) is one +row named after the folder above them, and `Dune.epub` beside `Dune.mobi` is +one row. + +**Adopting registers the files where they are.** Nothing is moved, renamed or +queued, and no indexer search starts. It is the scan's own match with you +supplying the answer, so **Undo** can take it back exactly. + +How to work through the list: + +- **A row with a suggestion** shows the closest book in your library and how + close the title is. **Confirm** adopts it in one click. +- **Other book** or **Choose book** opens the row in place: the suggestions + with their scores, a search of your library (prefilled from the file), and + a collapsed **Search metadata**. Metadata providers are only asked when you + press Search there, so opening rows never spends provider quota. **Add and + adopt** adds the book from metadata and adopts the files in one step. +- **The folder rail** lists the author folders with the most books to decide. + When no book in a folder matched because its author is not in your library, + that is one decision for the whole folder: **Add author**, then **Scan now**, + and the scan attaches what it can by itself. **Show** filters the list to a + folder, and **Ignore folder** sets all of its books aside. +- **Ignore** hides a row that is not a book you want tracked. Later scans keep + it hidden. The **Ignored** list brings any of them back. +- The sentence on each row says what the scan found and what to do next. + Hovering it shows the scanner's reason code, for bug reports. +- Keyboard: arrow keys move between rows, **Enter** opens one, **Esc** closes + it, **i** ignores, **u** undoes, **/** jumps to the search. + +What adopting does to your library: + +- Choosing a book **already in your library** attaches the files to it and + changes nothing else: its owner and its monitored flag stay as they were. +- A book added from metadata is added **unmonitored**, in the format you + adopted (ebook or audiobook), so Bindery never goes looking for the other + format behind your back. It and a new author are owned by the admin who + adopted them. The author's other books are not added. +- **Undo** removes exactly the file entries the adoption made. A book or author + the adoption created is removed too, unless something else now depends on + it (another file, another adopted row). +- Only an admin can see or act on this list, because it shows server paths. + +Things worth knowing: + +- Only regular files inside your library folders are listed. A symlink is not + adopted, including one inside the library that points elsewhere. +- A scan that finds no files at all (an unmounted volume, say) changes + nothing on this list, so your ignores and adoptions survive it. +- Adopted and ignored rows are forgotten 30 days after a scan last saw their + files unmatched. +- One scan lists up to 20,000 books from up to 50,000 unmatched files. A + larger library says so; adopt or ignore some and scan again. +- **From a folder** (`/import?view=folder`) is the other way in: point it at a + folder anywhere Bindery can read, such as your downloads, and it imports + what it matches into the library, moving or copying the files. + ## Metadata: where book data comes from - **OpenLibrary** is the default primary provider — it decides what an @@ -544,9 +609,10 @@ the list when you refresh the author, not on their own ([#2236](https://github.com/vavallee/bindery/issues/2236)). **Scan Library sees my files but imports nothing.** -Rule 1 — the catalogue is empty or the authors don't exist yet. Populate +Rule 1: the catalogue is empty or the authors don't exist yet. Populate first ([Bringing in an existing library](#bringing-in-an-existing-library)), -then scan. +then scan, or adopt the books from **Import → In your library** +([Adopting files](#adopting-files-already-in-your-library)). **I moved my files and a book still shows the old path.** Fixed (#2186). A book now shows whichever of its tracked files still exists, @@ -610,7 +676,9 @@ Rule 5 — separate mounts. One shared parent mount, then `auto` or `hardlink` mode. **Where do I drop files for Bindery to pick up?** -Nowhere (rule 3). Use Manual Import (`/import`) for files it didn't download. +Nowhere (rule 3). Use **Import** (`/import`) for files it didn't download: +**From a folder** for files elsewhere, **In your library** for files a library +scan found but could not match. --- From 5357e6189a277cbfe0570118d02eb46cf3927b57 Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 02:27:19 -0300 Subject: [PATCH 11/18] test(web): name the nav badge suite without a dash Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- web/src/App.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/App.test.tsx b/web/src/App.test.tsx index f629673ae..8a9841d15 100644 --- a/web/src/App.test.tsx +++ b/web/src/App.test.tsx @@ -286,7 +286,7 @@ describe('Shell — mobile navigation', () => { }) }) -describe('Shell — Import nav badge', () => { +describe('Shell: Import nav badge', () => { // Admins see how many library books still need a decision on the Import // entry. The count comes from an admin only route, so nobody else asks. it('shows the unmatched count for an admin', async () => { From 21126ad649f7a2659e456d942ab3407dfbaad42e Mon Sep 17 00:00:00 2001 From: vavallee Date: Thu, 17 Sep 2026 12:18:59 -0300 Subject: [PATCH 12/18] feat(web): calmer adoption list with groups, strong matches and fixed actions Books whose author is not in the library and share a folder are one group row with a single Add author and a disclosure for the books, and the rail is plain folder navigation. Only a strong suggestion (title similarity of at least 0.92 with a matching author, one constant in adoptionMatch.ts) gets a one click Confirm; anything weaker is a quiet "Possible match" link that opens the editor preselected. Every row has the same fixed action cell, one primary button and a More menu, and a one line fact instead of a wrapping sentence, with the full sentence in the tooltip and the editor. The summary is one number with quiet stats, sort arrows are SVG, counts use tabular numerals, and the library search no longer repeats the metadata search prompt. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- web/src/components/SortHeader.tsx | 24 ++- web/src/components/import/BookPicker.tsx | 2 +- web/src/i18n/locales/en.json | 60 ++++-- web/src/pages/import/AdoptionEditor.tsx | 11 +- web/src/pages/import/AdoptionGroupRow.tsx | 73 +++++++ web/src/pages/import/AdoptionRail.tsx | 103 ++++------ web/src/pages/import/AdoptionRow.tsx | 213 ++++++++++----------- web/src/pages/import/AdoptionSummary.tsx | 85 ++++---- web/src/pages/import/AdoptionTable.tsx | 163 ++++++++-------- web/src/pages/import/AdoptionView.test.tsx | 63 ++++-- web/src/pages/import/AdoptionView.tsx | 5 +- web/src/pages/import/adoptionGroups.ts | 77 ++++++++ web/src/pages/import/adoptionHint.test.ts | 28 ++- web/src/pages/import/adoptionHint.ts | 17 +- web/src/pages/import/adoptionMatch.ts | 58 ++++++ web/src/pages/import/adoptionStyles.ts | 16 ++ 16 files changed, 637 insertions(+), 361 deletions(-) create mode 100644 web/src/pages/import/AdoptionGroupRow.tsx create mode 100644 web/src/pages/import/adoptionGroups.ts create mode 100644 web/src/pages/import/adoptionMatch.ts create mode 100644 web/src/pages/import/adoptionStyles.ts diff --git a/web/src/components/SortHeader.tsx b/web/src/components/SortHeader.tsx index 34ec9dacf..04da6527e 100644 --- a/web/src/components/SortHeader.tsx +++ b/web/src/components/SortHeader.tsx @@ -13,9 +13,21 @@ interface Props { className?: string } -// SortHeader is a clickable column header with an arrow for the active sort. -// BooksPage and AuthorDetailPage each carry their own copy; this is the third, -// so it lives here, and moving those two onto it is a separate change. +// Chevrons drawn as SVG in the same outline style as the header icons, so the +// arrow never depends on a font having the glyph. +function SortIcon({ state }: { state: 'asc' | 'desc' | 'none' }) { + return ( + + ) +} + +// SortHeader is a clickable column header showing the active sort. BooksPage +// and AuthorDetailPage each carry their own copy; this is the third, so it +// lives here, and moving those two onto it is a separate change. // // Tailwind v4's Preflight gives ) diff --git a/web/src/components/import/BookPicker.tsx b/web/src/components/import/BookPicker.tsx index 3d2b174d4..fcc08488d 100644 --- a/web/src/components/import/BookPicker.tsx +++ b/web/src/components/import/BookPicker.tsx @@ -70,7 +70,7 @@ const BookPicker = forwardRef(function BookPicker({ onP {loading &&

{t('common.loading', 'Loading...')}

} {!loading && term.trim().length >= 2 && results.length === 0 && (

- {t('manualImport.noResults', 'No matching books in your library. Search metadata below to add it.')} + {t('manualImport.noResults', 'No matching books in your library.')}

)} {results.map(b => ( diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index d1935b373..090b01a0b 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -130,7 +130,7 @@ "failed": "Failed: {{error}}", "searchLabel": "Search your library", "searchPlaceholder": "Search your library for the book…", - "noResults": "No matching books in your library. Search metadata below to add it.", + "noResults": "No matching books in your library.", "metadataOpen": "Not in your library? Search metadata", "metadataLabel": "Search metadata", "metadataPlaceholder": "Title, author, ISBN, or ASIN", @@ -169,20 +169,20 @@ "no_title_parsed": "No title read" }, "confirm": "Confirm", - "other": "Other book", "choose": "Choose book", "ignore": "Ignore", "unignore": "Unignore", "undo": "Undo", "hint": { "reasonTooltip": "Scan reason: {{code}}", - "suggestion": "Closest match is {{title}} by {{author}} ({{percent}}%). Confirm it or choose another book.", "authorMissing": "{{author}} is not in your library yet. Add the author, then scan again.", "authorUnreadable": "No author could be read from these files. Choose the book they are.", "noCandidates": "{{author}} has no book in your library waiting for these files. Choose the book or search metadata.", "noTitle": "No title could be read from the file names or tags. Choose the book they are.", "noTitleMatch": "No book by {{author}} has a title close to this one. Choose the book it is.", - "noTitleMatchNoAuthor": "No book in your library has a title close to this one. Choose the book it is." + "noTitleMatchNoAuthor": "No book in your library has a title close to this one. Choose the book it is.", + "strong": "Strong match: {{title}} by {{author}}. Confirm it or choose another book.", + "possible": "Possible match: {{title}} by {{author}}, {{percent}}% title similarity. Check it, then adopt it or choose another book." }, "row": { "files_one": "{{count}} file", @@ -216,8 +216,8 @@ }, "col": { "book": "Book", - "why": "Why it is here", - "match": "Best match" + "match": "Best match", + "actions": "Actions" }, "caption": { "pending": "Books in your library that need a decision", @@ -228,13 +228,9 @@ "rail": { "label": "Folders with the most books to decide", "heading": "Folders", - "counts_one": "{{count}} book, {{files}} files", - "counts_other": "{{count}} books, {{files}} files", "notInLibrary": "{{author}} is not in your library", - "addAuthor": "Add author", - "show": "Show", - "showAll": "Show all", - "ignore": "Ignore folder" + "ignore": "Ignore folder", + "all": "All folders" }, "ignoreFolder": { "title": "Ignore every book in {{folder}}?", @@ -244,19 +240,22 @@ "summary": { "label": "Library scan summary", "scanning": "Scanning your library…", - "pending_one": "{{count}} book needs a decision, {{files}} files", - "pending_other": "{{count}} books need a decision, {{files}} files", "none": "Nothing needs a decision", - "lastScan": "Last scan {{when}}", - "neverScanned": "Not scanned yet", - "ignored": "{{count}} ignored", - "adopted": "{{count}} adopted", "nudge": "Author added. Scan now to match their files.", "scanningButton": "Scanning…", "scanNow": "Scan now", "scanError": "The last scan did not finish: {{error}}", "noFiles": "The last scan found no book files. Check that the library folder is mounted, then scan again. Your decisions were kept.", - "truncated": "This library is larger than one scan lists. Adopt or ignore some books and scan again to see the rest." + "truncated": "This library is larger than one scan lists. Adopt or ignore some books and scan again to see the rest.", + "needDecision_one": "book needs a decision", + "needDecision_other": "books need a decision", + "files_one": "file", + "files_other": "files", + "never": "never", + "lastScanLabel": "last scan", + "adoptedLabel": "adopted", + "ignoredLabel": "ignored", + "justNow": "just now" }, "empty": { "filteredTitle": "No books match these filters", @@ -271,6 +270,29 @@ "scanLibrary": "Scan library", "doneTitle": "Every book in your library is matched", "doneBody": "Nothing needs a decision. New files show up here after the next scan." + }, + "addAuthor": "Add author", + "showFiles": "Show files", + "more": "More", + "moreFor": "More actions for {{name}}", + "match": { + "strong": "Strong match", + "possible": "Possible match" + }, + "short": { + "authorMissing": "{{author}} is not in your library", + "authorUnreadable": "No author could be read", + "noCandidates": "No book by {{author}} is waiting for a file", + "noTitle": "No title could be read", + "noTitleMatch": "No close title by {{author}}", + "noTitleMatchNoAuthor": "No close title in your library" + }, + "group": { + "label": "{{author}}, {{count}} books", + "counts": "{{count}} books, {{files}} files", + "sentence": "Author not in your library yet", + "show": "Show books", + "hide": "Hide books" } }, "search": { diff --git a/web/src/pages/import/AdoptionEditor.tsx b/web/src/pages/import/AdoptionEditor.tsx index 058a7eb3e..4fe7c2464 100644 --- a/web/src/pages/import/AdoptionEditor.tsx +++ b/web/src/pages/import/AdoptionEditor.tsx @@ -4,12 +4,14 @@ import type { AdoptionBookRef, AdoptionItem, AdoptTarget, Book } from '../../api import { btn, btnSize } from '../../components/buttons' import BookPicker from '../../components/import/BookPicker' import CatalogueAdder from '../../components/import/CatalogueAdder' -import { scorePercent, unitDisplayName } from './adoptionHint' +import { adoptionHint, scorePercent, unitDisplayName } from './adoptionHint' interface Props { item: AdoptionItem onAdopt: (target: AdoptTarget, preview: AdoptionBookRef | null) => void | Promise onCancel: () => void + // Opens the file list, for the row's Show files action. + showFiles?: boolean } function refFromBook(b: Book): AdoptionBookRef { @@ -23,7 +25,7 @@ function refFromBook(b: Book): AdoptionBookRef { // suggestions as radio rows with their scores, a search of the library, // prefilled and focused, and a collapsed metadata search that asks a provider // only when submitted. Esc closes it and focus goes back to the row. -export default function AdoptionEditor({ item, onAdopt, onCancel }: Props) { +export default function AdoptionEditor({ item, onAdopt, onCancel, showFiles = false }: Props) { const { t } = useTranslation() const headingId = useId() const searchRef = useRef(null) @@ -32,7 +34,7 @@ export default function AdoptionEditor({ item, onAdopt, onCancel }: Props) { const [format, setFormat] = useState(item.format) const name = unitDisplayName(item) - useEffect(() => { searchRef.current?.focus() }, []) + useEffect(() => { searchRef.current?.focus({ preventScroll: showFiles }) }, [showFiles]) const onKeyDown = (e: KeyboardEvent) => { if (e.key === 'Escape') { @@ -57,6 +59,7 @@ export default function AdoptionEditor({ item, onAdopt, onCancel }: Props) {

{t('adoption.editor.heading', { name, defaultValue: 'Which book is {{name}}?' })}

+

{adoptionHint(item, t).sentence}

{options.length > 0 && (
@@ -134,7 +137,7 @@ export default function AdoptionEditor({ item, onAdopt, onCancel }: Props) { {item.members.length > 1 && ( -
+
{t('adoption.editor.files', { count: item.fileCount, defaultValue: '{{count}} files' })} diff --git a/web/src/pages/import/AdoptionGroupRow.tsx b/web/src/pages/import/AdoptionGroupRow.tsx new file mode 100644 index 000000000..e367b3eb9 --- /dev/null +++ b/web/src/pages/import/AdoptionGroupRow.tsx @@ -0,0 +1,73 @@ +import { forwardRef, type KeyboardEvent } from 'react' +import { useTranslation } from 'react-i18next' +import { btn, btnSize } from '../../components/buttons' +import MoreMenu from '../../components/MoreMenu' +import type { AdoptionGroup } from './adoptionGroups' +import { actionCellCls, cellCls, rowCls } from './adoptionStyles' + +interface Props { + group: AdoptionGroup + open: boolean + focusable: boolean + onFocusRow: () => void + onKeyDown: (e: KeyboardEvent) => void + onToggle: () => void + onAddAuthor: (name: string) => void + onIgnoreFolder: (folder: string) => void +} + +// AdoptionGroupRow stands for every book on the page under one author folder +// whose author is not in the library. It says so once, offers Add author once, +// and discloses the books, which keep their own Choose book and Ignore. +const AdoptionGroupRow = forwardRef(function AdoptionGroupRow( + { group, open, focusable, onFocusRow, onKeyDown, onToggle, onAddAuthor, onIgnoreFolder }, ref, +) { + const { t } = useTranslation() + const panelLabel = open ? t('adoption.group.hide', 'Hide books') : t('adoption.group.show', 'Show books') + return ( + { if (e.target === e.currentTarget) onFocusRow() }} + className={rowCls(false, false)} + > + + + + +

+ {t('adoption.group.sentence', 'Author not in your library yet')} +

+ + +
+ + onIgnoreFolder(group.folder) }, + ]} + /> +
+ + + ) +}) + +export default AdoptionGroupRow diff --git a/web/src/pages/import/AdoptionRail.tsx b/web/src/pages/import/AdoptionRail.tsx index 712fbf359..7da804767 100644 --- a/web/src/pages/import/AdoptionRail.tsx +++ b/web/src/pages/import/AdoptionRail.tsx @@ -3,90 +3,55 @@ import type { AdoptionFolderFacet } from '../../api/client' interface Props { folders: AdoptionFolderFacet[] + total: number activeFolder: string onShow: (folder: string) => void - onAddAuthor: (name: string) => void - onIgnoreFolder: (folder: string) => void } -// The structure rail: the author folders holding the most undecided books, -// each with the one action its books call for. When every book in a folder -// failed because its author is not in the library, that is one decision for -// the whole folder (add the author, then scan), not a decision per file. -// Anything else is a folder worth looking at, so it filters the list. +// The folder rail is navigation: each author folder with the most undecided +// books is one item that filters the list, its count right aligned. An amber +// dot marks a folder whose author is not in the library, which the list then +// shows as one group with Add author. Actions live in the list, not here. // // A column beside the list on wide screens, a row of scrolling chips on narrow // ones. -export default function AdoptionRail({ folders, activeFolder, onShow, onAddAuthor, onIgnoreFolder }: Props) { +export default function AdoptionRail({ folders, total, activeFolder, onShow }: Props) { const { t } = useTranslation() if (folders.length === 0) return null + const item = (key: string, label: string, count: number, dot: 'amber' | 'slate' | null, active: boolean, onClick: () => void, title?: string) => ( +
  • + +
    • + ) + return (