diff --git a/CLAUDE.md b/CLAUDE.md index dd113b1..78660ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -157,6 +157,11 @@ internal/ -> rescan -> ASIN enrich audiblesync/ Phase 4c: audible.sync (lastpositions -> furthest-only events) + the push loop (feed -> SetLastPosition) importjob/ BookMatcher (link.LibraryBook) + Plan: match and place against a library + -- LibriVox L6 -- + librivoxjob/ librivox.fetch (download lane): the manifest from meta's LibriVox-shaped + feed, the sections from archive.org only (https, every redirect checked) + into a staging folder, Author/Title/NN - Section.mp3 + cover.jpg + + .audiosilo/librivox.json by one no-replace rename, rescan transfer/ Local.Place: SafeJoin + os.Root, free-space check, no-replace install placement/ source/ progress/ (the manager's; source trimmed to Record in 4b) web/ the page: Vite + React 19 + TS + Tailwind v4; pure logic in src/lib @@ -166,7 +171,7 @@ Dockerfile slim | cuda | agent | full; compose.example.yaml: server, bridge, **Dependency direction:** `app -> {api, jobs, link, mapping, verify, generate, agentbox, workers, accounts, setup, correct, store, secrets, config, auth, logring, -web, netguard, podcast, podcastjob, audibleacct, audiblejob, audiblesync}` plus the engine it wires; `accounts -> {agentproto, ghauth, secrets, store}`; `correct -> +web, netguard, podcast, podcastjob, audibleacct, audiblejob, audiblesync, librivoxjob}` plus the engine it wires; `accounts -> {agentproto, ghauth, secrets, store}`; `correct -> {contrib, jobs, metaops}`; `setup -> link`; `api` calls the injected Backend and the injected `auth.Manager` (login, sessions, the password), with types from jobs/link/mapping/store/logring; `verify -> {jobs, @@ -177,7 +182,8 @@ through its own Engine/Server interfaces; `podcastjob -> {jobs, link, mapping, n imports `audible`); `importjob -> {link, placement, source, audiosilo-server/pkg/match}`; `audibleacct -> {audible, link, secrets, source, store}`; `audiblejob -> {audible, audibleacct, fsutil, importjob, jobs, link, mapping, placement, -source, store, transfer}` (app wires both); `audiblesync -> {audible, audibleacct, jobs, link, store}`; `transfer -> {fsutil, mapping, progress}`; +source, store, transfer}` (app wires both); `librivoxjob -> {fsutil, importjob, jobs, link, +mapping, metaops, netguard, placement, transfer, audiosilo-server/pkg/match}`; `audiblesync -> {audible, audibleacct, jobs, link, store}`; `transfer -> {fsutil, mapping, progress}`; `jobs -> {link, store}`; `mapping -> {fsutil, link}`; `link`, `agentproto` and `workerproto` are leaves; `workers -> {asr, fsutil, scheduler, state, store, transcript, workerproto}`; `worker -> {asr, fsutil, workerproto}` @@ -205,7 +211,10 @@ in PACKAGES.md. book writer is `generate` (its three jobs): `characters.json`, `recaps.json`, `description.json` (Phase 1c), `chapters.json` (Phase 1d), `readalong.json` (Phase 3c-1) and `bridge.json` in the book's `.audiosilo/` (a single-file book's prefixed - `.`). Podcast writers + `.`). `librivox.fetch` (LibriVox L6) writes whole new book folders: its own + dot staging folder beside the destination (removed outright on a cancel or a + failure: it holds only the job's own downloads), then one no-replace rename, + never into an existing book. Podcast writers require a podcast-kind library and its checked marker mapping (a job claimed before the check waits for it); they write only the root marker, show folders, feed.json and sniffed JPEG/PNG artwork. Book writers refuse podcast libraries. @@ -440,6 +449,14 @@ key set) with Range, a cut and an expiry. No migration. server's still places back to Audible; a refused account neither syncs nor pushes, a sync error never refuses one; the page's sync switch went (the console owns it). +**LibriVox L6-1 (Add from LibriVox, Bridge side):** `librivox.fetch` +(`librivoxjob`, LIBRIVOX-PLAN.md L6): the manifest from the Bridge's own meta base, +the sections from archive.org only (`AllowedURL` on every URL and redirect through +netguard's new per-request `Allow`), staged in the library and placed by one +no-replace rename with `cover.jpg` and `.audiosilo/librivox.json`, then rescanned; +resumable per section, its plan and placement marker in `/librivox/`. The +server records the book's LibriVox identity from the job's result (its fold). + See BRIDGE-PLAN.md's Status table for what is next. Operative caveats carried from sidecars (PACKAGES.md has the detail): diff --git a/PACKAGES.md b/PACKAGES.md index 6105e54..dafe9f5 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -1734,7 +1734,12 @@ internal/ landing in between is never cleared nor parked over. metaops/ meta.audiosilo.app client (coverage/lookup, capped 1h TTL caches, graceful degrade) + the PathAllowed/AllowedPath containment checks - (Phase 1a passes the mapped mounts as the roots). glossary.go adds + (Phase 1a passes the mapped mounts as the roots). librivox.go (LibriVox + L6): LibrivoxAudiobook (the LibriVox-shaped feed, one project with + extended=1&coverart=1; LibriVox's string ids, nullable coverart_jpg, + ArchiveID from url_iarchive) and LibrivoxWork (works/{id}: title, + author names, each recording's librivox_id and archive_id), uncached, + with ErrDisabled / ErrNotFound / ErrUnavailable apart. glossary.go adds SeriesGlossary (work -> its series -> the OTHER volumes' contributed character names/aliases, <= 12 siblings, own TTL caches), the canonical spelling evidence the spelling stage checks a transcript against; it @@ -2036,6 +2041,12 @@ internal/ failures, respecting Retry-After. Bad gzip data (empty, header, checksum, stream) fails at once as not_a_feed; a read the connection broke is retried. Errors never contain remote text or URLs. + LibriVox L6: Request and StreamRequest take an optional Allow, asked + about the first URL and every redirect's (CheckRedirect); a refusal is + blocked_host before the refused address is dialled. Nil keeps the + old rule (the podcasts pass none). + StreamResponse carries the Content-Type as sent (librivox.fetch refuses + a page where a file was asked for). podcast/ pure feed.json v1 types, rooted regular-file IO with capped atomic writes, URL normalization/hash/privacy/redaction, NFC show names, stable episode paths, identity merge and adaptive polling. Spike 5 @@ -2589,12 +2600,115 @@ internal/ the library holds a record is BookMatcher's). Run, the existence check and ApplyDestinations went in Phase 4b: audiblejob's import runs the steps. The manager's serverapi client went: the Bridge holds - the paged listing. Deps: link, placement, source, + the paged listing. AllBooks (books.go, LibriVox L6) reads that listing + whole for audiblejob and librivoxjob. Deps: link, placement, source, audiosilo-server/pkg/match. placement/ where a new book goes (the manager's): Auto (series siblings' folder and shortcode convention, else author/title) and templates. Phase 4b: a generated segment loses its leading dots too ("...And Ladies of the Club", ".hack"): a hidden name is never indexed. + LibriVox L6 exports Sanitize (librivoxjob names its section files + with it). + librivoxjob/ LibriVox L6 (LIBRIVOX-PLAN.md), librivox.fetch on the download lane + (capacity 1, shared with audible.import and an ordinary podcast.fetch: + one book at a time). Params + {librivox_id, work_id, recording_id, library_id} (server-built, + strict; unknown fields ignored). Manifest from the Bridge's OWN meta + base (metaops.LibrivoxAudiobook: /api/v1/librivox/audiobooks?id=N& + extended=1&coverart=1, LibriVox's shape; decision 5, the server's + mirror is never asked) and the work (metaops.LibrivoxWork, works/{id}; + a 404 there is no failure, the project names the folder then). meta + unavailable (5xx, a dropped connection) is never terminal: each read + is tried 4 times (5, 10, 20 s apart), then the run fails uncoded + (keepStaging) with its staging folder and state kept for the next + attempt; only meta's definite 404 ends the fetch as not_found. Every + section URL must be https on archive.org or *.archive.org + (AllowedURL, checked before anything is fetched or written, and on + every redirect through netguard's per-request Allow; never + librivox.org; a padded URL is refused as it stands, so the check + and the request see one spelling), fetched by the app's own netguard client (public + addresses only, not the podcasts' allow_networks). Placement: + placement.Suggest standalone (an existing author folder from + importjob.AuthorFolders reused verbatim, else the sanitized author), + NOT importjob.Planner.Plan: its match.CleanTitle drops "(version 2)" + and "Vol. 001", which tell two projects of one work apart. Title = + the work's, or the project's own when the work has more than one + LibriVox recording; author = the first credit, Various over three; + no author places at the library root; nothing left of a title is + "LibriVox ". Names: control characters and space runs folded, + Sanitize, cut to 180 bytes on a rune boundary; files "NN -
." (NN the 1-based position, at least two digits, the + count's width; never section_number; a title opening with the + section's own number and a separator ("1 - ", "1. ", "1: ", "1) ", + matching its section_number or position) loses it, so 5495's "1 - + Chapter 01 ..." is "01 - Chapter 01 ...", while "1984" stays; an + untitled section, or one that was only its number, is "Section N"; the URL's audio extension, else .mp3). The title's + folder taken, once " (LibriVox <id>)", else exists; a folder + already holding this project's librivox.json is the book (rescan + only); an author folder that is a listed book or holds audio is + exists. Staging <parent>/.audiosilo-librivox-<id>/ (a dot folder the + scan skips), every write through the mount's os.Root (0755/0644): + each section to <name>.part, its strong ETag and size recorded in + the state before the body (the ETag only beside a stated size; a + body from its start removes the old partial BEFORE the new + validators are saved, so a stop between the two never leaves old + bytes a later 206 would be proven to continue), a resume asks + Range/If-Range (netguard.ResumeAction proves the 206; anything + unproven starts the file again), renamed to its name only at the + stated size and never empty, and only audio (a Content-Type other + than none, audio/*, octet-stream or application/ogg is never written; + a whole file whose first bytes (past zero padding in its first 4 KB) + are no ID3 tag, MPEG frame sync, OggS, fLaC or MP4 ftyp is removed; + both tried again like a dropped + connection, then archive_unavailable; a finished file is skipped on + resume); + three attempts in a row that add nothing fail archive_unavailable + (adding = past the MOST the partial has held, section's high-water + mark, so restarts from zero that only pass the last cut don't + count), as does a 4xx other than 408, 416 and 429 (tried again); + the library's free space 1.1x each file's rest, else disk_full. + The mapping is checked again before each section, before the cover + and marker, and before the rename (recheck): not writable parks; a + mapping moved to another mount fails not_writable (the staging + folder isn't there), and the library is opened per section, not + held across a park. cover.jpg best effort (only a + sniffed JPEG from archive.org), then .audiosilo/librivox.json + {version 1, librivox_id, work_id, recording_id, archive_id} (decision + 3), the mapping checked again (a resumed or late job parks + not_writable), the staging folder swept to this manifest's files + (sweep: a partial or a section an earlier run staged under a title + the manifest changed since is removed, a missing section fails; an + entry is the manifest's by its name or by os.SameFile with the file + found under that name, since a mount may list NFD for NFC, and the + sections are looked for again after the removals), + ONE fsutil.RenameNoReplace staging -> book folder (a + folder that raced in takes the fallback once, never replaced), then + the rescan (3 tries). State <data>/librivox/<library_id>-<id>.json, + one per library and project, so fetches of one project into two + libraries never touch each other's staging (state.go: the + library, parent and whether the fetch made it, destination (staging is + <parent>/.audiosilo-librivox-<id>, never stored), + the partials' validators, then placed + count + bytes): a resumed run + follows it rather than planning again, Abandon finds the staging + folder by it, and once placed it is the placement marker (a failed + rescan keeps the book and the state; the retry goes on from the + rescan). A placement the state never recorded (a kill between the + rename and the state's write) is adopted by the marker: staging + gone and the folder or its fallback holding this project is placed + (adopt), never fetched again. A stop or a lost claim keeps staging + and state; a cancel, a + failure or Abandon (not on a lost claim; only the abandoned job's own + library's state) removes + the staging folder, the author folder it made when empty, and the + state. A lost claim + never handed out again leaves its staging folder (hidden, unindexed). + Steps manifest, download (done/total = sections), cover, place, + rescan. Result {library_id, path (the rescan's), librivox_id, files, + bytes} (files and bytes: the sections and the cover); a failure is + Failure {code, message} as result and error, codes not_writable, + exists, disk_full, not_found, archive_unavailable, metadata_off, + never a URL. Deps: fsutil, importjob, jobs, link, mapping, metaops, + netguard, placement, transfer, audiosilo-server/pkg/match. source/ Record (normalized metadata of one importable book: the ASIN, UnavailableReason; Phase 4a: Authors (every credit), SeriesPosition as the source spells it, ReleaseDate YYYY-MM-DD or "") (source.go only; @@ -3553,5 +3667,6 @@ and the shared clip-floor predicate `qa.ClipStartInRange`/`qa.ClipStartFloorSec` `state` is pure. Phase 4a: `audibleacct -> {audible, link, secrets, source, store}` and `audiblejob -> {audible, audibleacct, fsutil, importjob, jobs, link, mapping, placement, source, store, transfer}` (Phase 4b's import; app wires both; -neither imports app or api). Handlers marshal DTOs and call into the injected packages; they +neither imports app or api); LibriVox L6: `librivoxjob -> {fsutil, importjob, jobs, link, +mapping, metaops, netguard, placement, transfer, audiosilo-server/pkg/match}` (app wires it). Handlers marshal DTOs and call into the injected packages; they hold no logic (state transitions live in `state`, dispatch in `scheduler`). diff --git a/internal/app/app.go b/internal/app/app.go index 3340a53..8493daf 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -75,6 +75,9 @@ type Bridge struct { runner *jobs.Runner auth *auth.Manager podcastHTTP *netguard.Client + // librivoxHTTP is librivox.fetch's guarded client: public addresses only + // (never the podcasts' allow_networks), archive.org hosts only per request. + librivoxHTTP *netguard.Client // box is the agent container runner (decision 15), gen the sidecars.generate // job, engine the sidecars engine once it has started. box *agentbox.Runner @@ -216,7 +219,8 @@ func New(ctx context.Context, opts Options) (*Bridge, string, error) { } return nil }}) - corrector := correct.New(correct.Options{Meta: metaops.NewClient(cfg.Metadata.BaseURL), Tokens: b.tokens, + meta := metaops.NewClient(cfg.Metadata.BaseURL) + corrector := correct.New(correct.Options{Meta: meta, Tokens: b.tokens, BaseURL: cfg.Contribution.APIBaseURL, CoreRepo: cfg.Contribution.CoreRepo, KV: db, ServerID: serverID, Log: opts.Logger}) var audiblePoll *audiblejob.Handler var audibleImport *audiblejob.Importer @@ -225,7 +229,7 @@ func New(ctx context.Context, opts Options) (*Bridge, string, error) { audibleSync, b.audiblePush = b.newAudibleSync(db, opts.Logger, serverID) b.runner = jobs.New(db, b.server, b.tokenRefused, jobs.Options{Log: opts.Logger, ServerID: serverID}, verifier, b.gen, b.gen.DescribeJob(), b.gen.ChaptersJob(), b.gen.ReadalongJob(), b.gen.ContributeJob(), corrector, podcasts, podcasts.PollJob(), podcasts.FetchJob(), podcasts.RetainJob(), - audiblePoll, audibleImport, audibleSync) + audiblePoll, audibleImport, audibleSync, b.newLibrivox(meta, opts.Logger)) conn.onChange = b.linkChanged return b, oneTime, nil } @@ -298,6 +302,7 @@ var listen = func(ctx context.Context, addr string) (net.Listener, error) { // or the page's server fails (its error is returned once everything stopped). func (b *Bridge) Run(ctx context.Context) error { defer b.podcastHTTP.Close() + defer b.librivoxHTTP.Close() defer func() { _ = b.db.Close() }() // The heartbeat and the runner stop with ctx: a failing page server must // stop them too, or Run would wait for them for ever. diff --git a/internal/app/app_test.go b/internal/app/app_test.go index aeebf01..02447bb 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -144,7 +144,7 @@ func TestPairBeatMapAndRevoke(t *testing.T) { return ok && len(st.Libraries) == 1 && st.Libraries[0].Mapped && st.Libraries[0].Writable }) st, _ := fs.lastBeat() - if st.Version != "0.1.0" || strings.Join(st.Kinds, ",") != "audible.import,audible.poll,audible.sync,bridge.verify,chapters.detect,meta.correct,podcast.fetch,podcast.poll,podcast.retain,podcast.subscribe,readalong.align,sidecars.contribute,sidecars.describe,sidecars.generate" || st.UIURL != "http://bridge.lan:8091" { + if st.Version != "0.1.0" || strings.Join(st.Kinds, ",") != "audible.import,audible.poll,audible.sync,bridge.verify,chapters.detect,librivox.fetch,meta.correct,podcast.fetch,podcast.poll,podcast.retain,podcast.subscribe,readalong.align,sidecars.contribute,sidecars.describe,sidecars.generate" || st.UIURL != "http://bridge.lan:8091" { t.Fatalf("heartbeat = %+v", st) } // Once the engine runs, every heartbeat carries the estimates. @@ -283,7 +283,7 @@ func TestEngineViewAndJobRoutes(t *testing.T) { if _, err := b.Job(context.Background(), 5); !errors.Is(err, api.ErrNotFound) { t.Fatalf("an unknown job = %v", err) } - if k := strings.Join(b.runner.Kinds(), ","); k != "audible.import,audible.poll,audible.sync,bridge.verify,chapters.detect,meta.correct,podcast.fetch,podcast.poll,podcast.retain,podcast.subscribe,readalong.align,sidecars.contribute,sidecars.describe,sidecars.generate" { + if k := strings.Join(b.runner.Kinds(), ","); k != "audible.import,audible.poll,audible.sync,bridge.verify,chapters.detect,librivox.fetch,meta.correct,podcast.fetch,podcast.poll,podcast.retain,podcast.subscribe,readalong.align,sidecars.contribute,sidecars.describe,sidecars.generate" { t.Fatalf("kinds = %s", k) } } diff --git a/internal/app/librivox.go b/internal/app/librivox.go new file mode 100644 index 0000000..329600e --- /dev/null +++ b/internal/app/librivox.go @@ -0,0 +1,26 @@ +package app + +import ( + "log/slog" + "path/filepath" + + "github.com/kodestar/audiosilo-bridge/internal/librivoxjob" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/netguard" +) + +// newLibrivox builds librivox.fetch (LIBRIVOX-PLAN.md L6): the manifest from +// the Bridge's own metadata base (decision 5), the files from archive.org +// through a guarded client of its own (decision 7), placed in a mapped books +// library and rescanned. Its state lives in <data>/librivox. +func (b *Bridge) newLibrivox(meta *metaops.Client, log *slog.Logger) *librivoxjob.Handler { + b.librivoxHTTP = netguard.New(netguard.Options{Version: b.version}) + return librivoxjob.New(librivoxjob.Options{Meta: meta, Fetcher: b.librivoxHTTP, Libraries: b.libs, Mount: b.mount, + StateDir: filepath.Join(b.dataDir, "librivox"), Log: log, + Server: func() librivoxjob.Server { + if cl := b.conn.Client(); cl != nil { + return cl + } + return nil + }}) +} diff --git a/internal/audiblejob/audiblejob.go b/internal/audiblejob/audiblejob.go index f199678..93d0892 100644 --- a/internal/audiblejob/audiblejob.go +++ b/internal/audiblejob/audiblejob.go @@ -39,9 +39,6 @@ import ( // Kind is the job kind. const Kind = "audible.poll" -// maxBookPages bounds the library listing (500 books a page: a million books). -const maxBookPages = 2000 - // maxRuntime is the longest runtime the server takes (seconds). const maxRuntime = 1_000_000 @@ -457,30 +454,12 @@ func (h *Handler) refused(ctx context.Context, id string) *Failure { return fail(CodeSignIn, "Audible refused the account's sign-in: the next check tries again, or sign in again on the Bridge's Accounts tab.") } -// bookLister reads a library's books page by page (Server and ImportServer). -type bookLister interface { - LibraryBooks(ctx context.Context, libraryID int64, after string, limit int) (*link.LibraryBooks, error) -} - -// allBooks reads every page of the library's books. -func allBooks(ctx context.Context, srv bookLister, libraryID int64) ([]link.LibraryBook, error) { +// allBooks reads every page of the account's library's books. +func allBooks(ctx context.Context, srv importjob.BookLister, libraryID int64) ([]link.LibraryBook, error) { if libraryID <= 0 { return nil, errors.New("the account has no library: map it again on the Bridge's Accounts tab") } - var books []link.LibraryBook - after := "" - for range maxBookPages { - page, err := srv.LibraryBooks(ctx, libraryID, after, link.MaxLibraryBooks) - if err != nil { - return nil, err - } - books = append(books, page.Books...) - if page.Next == "" || page.Next == after { - return books, nil - } - after = page.Next - } - return nil, fmt.Errorf("the library has more than %d pages of books", maxBookPages) + return importjob.AllBooks(ctx, srv, libraryID) } // title is a record as the poll reports it. diff --git a/internal/importjob/books.go b/internal/importjob/books.go new file mode 100644 index 0000000..6b9b7b9 --- /dev/null +++ b/internal/importjob/books.go @@ -0,0 +1,35 @@ +package importjob + +import ( + "context" + "fmt" + + "github.com/kodestar/audiosilo-bridge/internal/link" +) + +// maxBookPages bounds the library listing (500 books a page: a million books). +const maxBookPages = 2000 + +// BookLister reads a library's books page by page (*link.Client). +type BookLister interface { + LibraryBooks(ctx context.Context, libraryID int64, after string, limit int) (*link.LibraryBooks, error) +} + +// AllBooks reads every page of the library's books (the server's paged +// listing, read whole), as Plan and the matcher take them. +func AllBooks(ctx context.Context, srv BookLister, libraryID int64) ([]link.LibraryBook, error) { + var books []link.LibraryBook + after := "" + for range maxBookPages { + page, err := srv.LibraryBooks(ctx, libraryID, after, link.MaxLibraryBooks) + if err != nil { + return nil, err + } + books = append(books, page.Books...) + if page.Next == "" || page.Next == after { + return books, nil + } + after = page.Next + } + return nil, fmt.Errorf("the library has more than %d pages of books", maxBookPages) +} diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go new file mode 100644 index 0000000..5c4343e --- /dev/null +++ b/internal/librivoxjob/download.go @@ -0,0 +1,382 @@ +package librivoxjob + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "math" + "mime" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/kodestar/audiosilo-bridge/internal/fsutil" + "github.com/kodestar/audiosilo-bridge/internal/jobs" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/netguard" + "github.com/kodestar/audiosilo-bridge/internal/transfer" +) + +// downloadAttempts bounds a section's attempts in a row that add nothing to +// its partial (a dropped connection resumes from it each time; an attempt that +// added to it starts the count again). +const downloadAttempts = 3 + +// maxFileBytes caps one section's file (a LibriVox section is tens of MB). +const maxFileBytes int64 = 2 << 30 + +// maxCoverBytes caps the cover. +const maxCoverBytes int64 = 20 << 20 + +// maxRetryAfter bounds how long a Retry-After is waited out. +const maxRetryAfter = 5 * time.Minute + +// partSuffix names a section's partial download in the staging folder. +const partSuffix = ".part" + +// copyBufBytes is how much of a section's body is gathered before a write. +const copyBufBytes = 1 << 20 + +// download fetches every section not in the staging folder yet: a file there +// under its own name is finished (it is renamed from its .part only once +// whole), so a resumed run skips it and goes on with the next, resuming that +// one's .part with Range/If-Range when its strong ETag was recorded. +func (h *Handler) download(ctx context.Context, f *fetch, sections []metaops.LibrivoxSection, names []string) error { + n := len(sections) + for i, s := range sections { + // The mapping again before each section's writes: one that went away + // parks the job (not_writable) with what it fetched, until it is back. + if err := h.recheck(ctx, f, StepDownload); err != nil { + return err + } + h.report(f.rep, jobs.Progress{Step: StepDownload, Message: fmt.Sprintf("Downloading section %d of %d from archive.org", i+1, n), + Done: i, Total: n}) + if err := h.fetchSection(ctx, f, i, s.ListenURL, names[i]); err != nil { + return err + } + } + h.report(f.rep, jobs.Progress{Step: StepDownload, Message: "Downloaded from archive.org", Done: n, Total: n}) + return nil +} + +// fetchSection downloads section i unless the staging folder holds it already. +// The library is opened for each section: a handle opened before a park may +// be on a mount that went away since. +func (h *Handler) fetchSection(ctx context.Context, f *fetch, i int, rawURL, name string) error { + root, err := f.openRoot() + if err != nil { + return err + } + defer func() { _ = root.Close() }() + if st, err := root.Lstat(path.Join(f.staging(), name)); err == nil && st.Mode().IsRegular() { + return nil + } + return h.section(ctx, root, f, i, rawURL, name) +} + +// retry is an attempt that may be tried again: after retryAfter when the +// server said so. size is what the partial holds after it (0 when the attempt +// wrote nothing or the partial went). +type retry struct { + retryAfter int + size int64 + cause string +} + +func (r *retry) Error() string { return r.cause } + +// partSize is what a section's partial holds, 0 when there is none. +func partSize(root *os.Root, part string) int64 { + if st, err := root.Lstat(part); err == nil && st.Mode().IsRegular() { + return st.Size() + } + return 0 +} + +// section downloads one section to <staging>/<name>, trying again while +// attempts add to the partial (downloadAttempts in a row that add nothing +// fail archive_unavailable). +func (h *Handler) section(ctx context.Context, root *os.Root, f *fetch, i int, rawURL, name string) error { + // high is the most the partial has held. Only an attempt that takes it + // past that added to it: one that started the file again from zero adds + // nothing until it does, however its size compares with the last cut's. + high := partSize(root, path.Join(f.staging(), name)+partSuffix) + failures := 0 + for { + err := h.attempt(ctx, root, f, i, rawURL, name) + var r *retry + switch { + case err == nil: + return nil + case ctx.Err() != nil: + return ctx.Err() + case !errors.As(err, &r): + return err + } + if r.size > high { + high, failures = r.size, 0 + } else { + failures++ + } + if failures >= downloadAttempts { + return fail(CodeArchiveUnavailable, "archive.org didn't deliver section %d of LibriVox recording %d (%s): try again later.", + i+1, f.p.LibrivoxID, r.cause) + } + h.o.Log.Info("librivox: a section's download failed; trying again", "librivox_id", f.p.LibrivoxID, "section", i+1, + "failures", failures, "cause", r.cause) + backoff := max(failures, 1) + wait := max(time.Duration(backoff*backoff)*2*time.Second, min(time.Duration(r.retryAfter)*time.Second, maxRetryAfter)) + if werr := h.o.Wait(ctx, wait); werr != nil { + return werr + } + } +} + +// attempt is one GET of a section: a 200 writes the file from its start, a +// 206 proven to continue the partial (netguard.ResumeAction) appends to it, +// anything unproven starts again. The file is renamed from its .part once it +// holds the size the response stated. +func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, rawURL, name string) error { + final := path.Join(f.staging(), name) + part := final + partSuffix + offset := partSize(root, part) + prev := f.st.Files[name] + if offset == 0 { + prev = partState{} + } + resp, err := h.o.Fetcher.Stream(ctx, netguard.StreamRequest{URL: rawURL, Offset: offset, ETag: prev.ETag, Allow: AllowedURL}) + if err != nil { + e := netguard.Classify(err) + if netguard.Retryable(err) { + return &retry{retryAfter: e.RetryAfterSeconds, cause: e.Code} + } + return fail(CodeArchiveUnavailable, "The Bridge can't download section %d of LibriVox recording %d from archive.org (%s).", + i+1, f.p.LibrivoxID, e.Code) + } + defer func() { _ = resp.Body.Close() }() + + appending := false + start, total := int64(0), int64(0) + switch { + case resp.Encoding != "" && resp.Encoding != "identity": + return fail(CodeArchiveUnavailable, "archive.org sent section %d of LibriVox recording %d encoded (%s).", i+1, f.p.LibrivoxID, resp.Encoding) + case resp.Status == http.StatusOK: + total = max(resp.ContentLength, 0) + case resp.Status == http.StatusPartialContent && netguard.ResumeAction(offset, prev.Total, prev.ETag, resp) == "append": + appending, start, total = true, offset, prev.Total + case resp.Status == http.StatusPartialContent, resp.Status == http.StatusRequestedRangeNotSatisfiable: + // Not a continuation the Bridge can prove: the partial goes and the + // next attempt asks for the whole file. + _ = root.Remove(part) + delete(f.st.Files, name) + return &retry{cause: fmt.Sprintf("http_%d", resp.Status)} + case resp.Status == http.StatusRequestTimeout, resp.Status == http.StatusTooManyRequests, resp.Status >= 500: + return &retry{retryAfter: resp.RetryAfterSeconds, cause: fmt.Sprintf("http_%d", resp.Status)} + default: + return fail(CodeArchiveUnavailable, "archive.org answered HTTP %d for section %d of LibriVox recording %d.", resp.Status, i+1, f.p.LibrivoxID) + } + if !audioType(resp.ContentType) { + // A page where a file was asked for (an error or a login page with a + // 200): never written, tried again like a dropped connection. + return &retry{size: offset, cause: "not_audio"} + } + if total > maxFileBytes { + return fmt.Errorf("section %d is larger than the Bridge's cap for one file", i+1) + } + // A body from its start replaces the partial, which goes before the new + // validators are recorded: a stop between the two must never leave the old + // bytes under validators that would prove a later 206 continues them. + if !appending { + if err := root.Remove(part); err != nil && !errors.Is(err, fs.ErrNotExist) { + return writeFailure(err, part) + } + } + if total > 0 { + need := int64(math.Ceil(transfer.DefaultHeadroom * float64(total-start))) + if free, err := h.o.Free(filepath.Join(f.mount, filepath.FromSlash(f.staging()))); err == nil && free < need { + return fail(CodeDiskFull, "The library is short of space for section %d of the recording.", i+1) + } + } + // The validators go into the state before the body: a stop during the + // body resumes this file from what it holds. + if f.st.Files == nil { + f.st.Files = map[string]partState{} + } + // The ETag only beside a stated size: without one no 206 could be proven + // to continue the partial, so the next attempt asks for the whole file. + ps := partState{Total: total} + if total > 0 && netguard.StrongETag(resp.ETag) { + ps.ETag = resp.ETag + } + f.st.Files[name] = ps + if err := h.saveState(f.p.LibrivoxID, f.st); err != nil { + return fmt.Errorf("recording a download's progress in the Bridge's data: %w", err) + } + + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC + if appending { + flags = os.O_WRONLY | os.O_CREATE | os.O_APPEND + } + file, err := root.OpenFile(part, flags, 0o644) + if err != nil { + return writeFailure(err, part) + } + // Buffered: a body arrives in reads of a TLS record at most, and each as + // its own write is a round trip on a network mount. The flush runs after a + // cut connection too, so the partial holds all that arrived. + w := &writeErr{w: file} + buf := bufio.NewWriterSize(w, copyBufBytes) + n, cerr := io.Copy(buf, io.LimitReader(resp.Body, maxFileBytes-start+1)) + if err := buf.Flush(); err != nil && w.err == nil { + w.err = err + } + if err := file.Sync(); err != nil && w.err == nil { + w.err = err + } + if err := file.Close(); err != nil && w.err == nil { + w.err = err + } + size := start + n + switch { + case w.err != nil: + return writeFailure(w.err, part) + case size > maxFileBytes: + _ = root.Remove(part) + return fmt.Errorf("section %d is larger than the Bridge's cap for one file", i+1) + case cerr != nil: + return &retry{size: size, cause: "network"} + case size == 0, total > 0 && size != total: + // An empty body is never a section, whatever length the response + // stated; one longer than stated can't be trusted either. + if size > total { + _ = root.Remove(part) + delete(f.st.Files, name) + size = 0 + } + return &retry{size: size, cause: "short"} + } + if !audioHead(root, part) { + // Whole but not audio: it goes, and the section starts again. + _ = root.Remove(part) + delete(f.st.Files, name) + return &retry{cause: "not_audio"} + } + if err := root.Rename(part, final); err != nil { + return writeFailure(err, final) + } + // The file under its own name is the record that it is finished: its + // entry goes with the next save. + delete(f.st.Files, name) + return nil +} + +// audioType reports whether a response's Content-Type may carry a section: +// none stated, audio/*, or a generic binary type. A text, HTML, JSON or other +// typed answer is a page, not the file. +func audioType(contentType string) bool { + if strings.TrimSpace(contentType) == "" { + return true + } + // A malformed parameter ("audio/mpeg; charset") still names the type: + // ParseMediaType returns it beside ErrInvalidMediaParameter. + mt, _, err := mime.ParseMediaType(contentType) + if err != nil && !errors.Is(err, mime.ErrInvalidMediaParameter) { + return false + } + return strings.HasPrefix(mt, "audio/") || mt == "application/octet-stream" || mt == "binary/octet-stream" || + mt == "application/ogg" +} + +// audioHead reports whether a finished section's first bytes are audio: an +// ID3 tag or an MPEG frame sync (LibriVox's MP3s), else the Ogg, FLAC or MP4 +// signature a section of another extension starts with. Zero bytes before +// the tag or the first frame (padding some encoders leave) are skipped within +// the first headWindow bytes. +func audioHead(root *os.Root, name string) bool { + file, err := root.Open(name) + if err != nil { + return false + } + defer func() { _ = file.Close() }() + head := make([]byte, headWindow) + n, _ := io.ReadFull(file, head) + return isAudioHead(head[:n]) +} + +// headWindow bounds how far audioHead looks past leading zero padding. +const headWindow = 4096 + +func isAudioHead(b []byte) bool { + if len(b) >= 8 && string(b[4:8]) == "ftyp" { + // MP4's box size leads with zero bytes: checked before the padding goes. + return true + } + b = bytes.TrimLeft(b, "\x00") + switch { + case bytes.HasPrefix(b, []byte("ID3")), bytes.HasPrefix(b, []byte("OggS")), bytes.HasPrefix(b, []byte("fLaC")): + return true + case len(b) >= 2 && b[0] == 0xFF && b[1]&0xE0 == 0xE0: + return true + } + return false +} + +// writeErr remembers a write's failure, so a failed copy tells the library's +// fault (disk_full, not_writable) from the connection's (try again). +type writeErr struct { + w io.Writer + err error +} + +func (w *writeErr) Write(p []byte) (int, error) { + n, err := w.w.Write(p) + if err != nil { + w.err = err + } + return n, err +} + +// cover fetches the project's cover into the staging folder as cover.jpg: +// best effort (a book without one is still a book), only from archive.org, +// and only when it is a JPEG. +func (h *Handler) cover(ctx context.Context, f *fetch, book metaops.LibrivoxBook) { + if book.CoverartJPG == nil || *book.CoverartJPG == "" { + return + } + root, err := f.openRoot() + if err != nil { + return + } + defer func() { _ = root.Close() }() + if _, err := root.Lstat(path.Join(f.staging(), coverName)); err == nil { + return + } + h.report(f.rep, jobs.Progress{Step: StepCover, Message: "Downloading the cover from archive.org"}) + log := h.o.Log.With("librivox_id", f.p.LibrivoxID) + if !allowedRaw(*book.CoverartJPG) { + log.Info("librivox: the cover isn't on archive.org; the book goes without one") + return + } + resp, err := h.o.Fetcher.Fetch(ctx, netguard.Request{URL: *book.CoverartJPG, MaxBytes: maxCoverBytes, Allow: AllowedURL}) + switch { + case err != nil: + log.Info("librivox: the cover couldn't be downloaded; the book goes without one", "cause", netguard.Classify(err).Code) + return + case resp.Status != http.StatusOK || http.DetectContentType(resp.Body) != "image/jpeg": + log.Info("librivox: the cover isn't a JPEG; the book goes without one") + return + } + if err := fsutil.WriteRootAtomic(root, f.staging(), coverName, resp.Body, 0o644, nil); err != nil && !errors.Is(err, fs.ErrExist) { + log.Warn("librivox: writing the cover failed; the book goes without one", "err", err) + } +} + +// coverName is the cover's file in the book's folder. +const coverName = "cover.jpg" diff --git a/internal/librivoxjob/fake_test.go b/internal/librivoxjob/fake_test.go new file mode 100644 index 0000000..c9134db --- /dev/null +++ b/internal/librivoxjob/fake_test.go @@ -0,0 +1,368 @@ +package librivoxjob + +import ( + "bytes" + "cmp" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + "github.com/kodestar/audiosilo-bridge/internal/jobs" + "github.com/kodestar/audiosilo-bridge/internal/link" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/netguard" +) + +// fakeMeta answers the manifest and the work. +type fakeMeta struct { + book metaops.LibrivoxBook + bookErr error + work metaops.LibrivoxWork + workErr error + reads int + // bookErrs and workErrs: each read pops one first (nil: answer as usual). + bookErrs, workErrs []error +} + +func pop(q *[]error) error { + if len(*q) == 0 { + return nil + } + err := (*q)[0] + *q = (*q)[1:] + return err +} + +func (m *fakeMeta) LibrivoxAudiobook(_ context.Context, id int) (metaops.LibrivoxBook, error) { + m.reads++ + if err := pop(&m.bookErrs); err != nil { + return metaops.LibrivoxBook{}, err + } + if m.bookErr != nil { + return metaops.LibrivoxBook{}, m.bookErr + } + if m.book.ID != strconv.Itoa(id) { + return metaops.LibrivoxBook{}, metaops.ErrNotFound + } + return m.book, nil +} + +func (m *fakeMeta) LibrivoxWork(context.Context, string) (metaops.LibrivoxWork, error) { + if err := pop(&m.workErrs); err != nil { + return metaops.LibrivoxWork{}, err + } + return m.work, m.workErr +} + +// remote is one file the fake archive.org serves. +type remote struct { + body []byte + // ctype is the Content-Type sent ("" is audio/mpeg). + ctype string + etag string + status int // an error status instead of the file + // redirect: the file is served from this URL, which Allow is asked about + // first (netguard's CheckRedirect). + redirect string + // cut: each GET pops one; >= 0 delivers that many bytes of the answer and + // then fails the read (a dropped connection). + cut []int + // onCut runs when a cut fires (a test stops the job there). + onCut func() +} + +// request is one GET the fake saw. +type request struct { + url string + offset int64 + etag string +} + +// fakeFetcher is archive.org: Stream with strong-ETag ranges, Fetch for the +// cover, each URL and redirect through the request's Allow. +type fakeFetcher struct { + mu sync.Mutex + files map[string]*remote + requests []request +} + +func (f *fakeFetcher) seen() []request { + f.mu.Lock() + defer f.mu.Unlock() + return append([]request(nil), f.requests...) +} + +func (f *fakeFetcher) resolve(raw string, allow func(*url.URL) bool) (*remote, error) { + refused := func(at string) bool { + u, err := url.Parse(at) + return err != nil || (allow != nil && !allow(u)) + } + if refused(raw) { + return nil, &netguard.Error{Code: "blocked_host", Message: "blocked"} + } + r := f.files[raw] + if r == nil { + return &remote{status: http.StatusNotFound}, nil + } + if r.redirect != "" && refused(r.redirect) { + return nil, &netguard.Error{Code: "blocked_host", Message: "blocked"} + } + return r, nil +} + +type cutReader struct { + r io.Reader + left int + onCut func() +} + +func (c *cutReader) Read(p []byte) (int, error) { + if c.left <= 0 { + if c.onCut != nil { + c.onCut() + } + return 0, errors.New("connection reset") + } + if len(p) > c.left { + p = p[:c.left] + } + n, err := c.r.Read(p) + c.left -= n + return n, err +} + +func (f *fakeFetcher) Stream(_ context.Context, in netguard.StreamRequest) (*netguard.StreamResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.requests = append(f.requests, request{url: in.URL, offset: in.Offset, etag: in.ETag}) + r, err := f.resolve(in.URL, in.Allow) + if err != nil { + return nil, err + } + if r.status != 0 { + return &netguard.StreamResponse{Status: r.status, Body: io.NopCloser(bytes.NewReader(nil)), ContentLength: 0}, nil + } + resp := &netguard.StreamResponse{Status: http.StatusOK, ETag: r.etag, ContentLength: int64(len(r.body)), ContentType: cmp.Or(r.ctype, "audio/mpeg")} + body := r.body + // Range only with a strong ETag, as netguard sends it. + if in.Offset > 0 && netguard.StrongETag(in.ETag) && in.ETag == r.etag && in.Offset < int64(len(r.body)) { + resp.Status = http.StatusPartialContent + body = r.body[in.Offset:] + resp.ContentLength = int64(len(body)) + resp.ContentRange = fmt.Sprintf("bytes %d-%d/%d", in.Offset, len(r.body)-1, len(r.body)) + } + var rd io.Reader = bytes.NewReader(body) + if len(r.cut) > 0 { + n := r.cut[0] + r.cut = r.cut[1:] + if n >= 0 { + rd = &cutReader{r: rd, left: n, onCut: r.onCut} + } + } + resp.Body = io.NopCloser(rd) + return resp, nil +} + +func (f *fakeFetcher) Fetch(_ context.Context, in netguard.Request) (*netguard.Response, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.requests = append(f.requests, request{url: in.URL}) + r, err := f.resolve(in.URL, in.Allow) + if err != nil { + return nil, err + } + if r.status != 0 { + return nil, &netguard.Error{Code: "http_" + strconv.Itoa(r.status), Message: "status"} + } + return &netguard.Response{Status: http.StatusOK, Body: r.body}, nil +} + +// mp3 is an n-byte section body: an ID3 header, then fill. +func mp3(fill byte, n int) []byte { + return append([]byte("ID3"), bytes.Repeat([]byte{fill}, n-3)...) +} + +// fakeServer is the server link: the library's books and the rescan. +type fakeServer struct { + mu sync.Mutex + books []link.LibraryBook + rescanErr []error + rescanned []string + answer string // the rescan's book path ("" = the path asked) +} + +func (s *fakeServer) LibraryBooks(context.Context, int64, string, int) (*link.LibraryBooks, error) { + return &link.LibraryBooks{Books: s.books}, nil +} + +func (s *fakeServer) Rescan(_ context.Context, _ int64, p string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.rescanned = append(s.rescanned, p) + if len(s.rescanErr) > 0 { + err := s.rescanErr[0] + s.rescanErr = s.rescanErr[1:] + if err != nil { + return "", err + } + } + if s.answer != "" { + return s.answer, nil + } + return p, nil +} + +// fakeLibs is the mapping registry: library 3, books, mapped at mount. +type fakeLibs struct { + mount string + writable bool + kind string + // second, when set, is library 4's mount: a second books library. + second string +} + +// mountOf is library id's mount, "" for none. +func (l *fakeLibs) mountOf(id int64) string { + switch { + case id == 3: + return l.mount + case id == 4 && l.second != "": + return l.second + } + return "" +} + +func (l *fakeLibs) Library(id int64) (link.Library, bool) { + if l.mountOf(id) == "" { + return link.Library{}, false + } + return link.Library{ID: id, Name: "Books", Root: "/srv/books" + strconv.FormatInt(id, 10), Kind: l.kind}, true +} + +func (l *fakeLibs) Status(id int64) (link.LibraryStatus, bool) { + mount := l.mountOf(id) + if mount == "" { + return link.LibraryStatus{}, false + } + return link.LibraryStatus{LibraryID: id, MountPath: mount, Mapped: true, Writable: l.writable}, true +} + +// harness is a handler over the fakes, a library folder and a state folder. +type harness struct { + h *Handler + meta *fakeMeta + fetch *fakeFetcher + srv *fakeServer + libs *fakeLibs + mount string + stateDir string + free int64 + progress []jobs.Progress + mu sync.Mutex +} + +const ( + testID = 192 + testWork = "johnny-crows-garden-brooke" + testRec = "johnny-crows-garden-brooke-librivox-192" + testItem = "johnny_crow_garden_librivox" + testAuthor = "L. Leslie Brooke" + testTitle = "Johnny Crow's Garden" +) + +func sectionURL(n int) string { + return fmt.Sprintf("https://www.archive.org/download/%s/johnnycrow_%02d_brooke_64kb.mp3", testItem, n) +} + +const coverURL = "https://archive.org/download/" + testItem + "/johnnycrow_1104.jpg" + +// jpeg is the smallest prefix http.DetectContentType calls image/jpeg. +var jpeg = []byte("\xFF\xD8\xFFcover-bytes") + +func newHarness(t *testing.T) *harness { + t.Helper() + x := &harness{mount: t.TempDir(), stateDir: filepath.Join(t.TempDir(), "librivox"), free: 1 << 40} + cover := coverURL + x.meta = &fakeMeta{ + book: metaops.LibrivoxBook{ID: strconv.Itoa(testID), Title: testTitle, + Authors: []metaops.LibrivoxPerson{{FirstName: "L. Leslie", LastName: "Brooke"}}, + URLIArchive: "https://www.archive.org/details/" + testItem, CoverartJPG: &cover, + Sections: []metaops.LibrivoxSection{ + {Title: "Johnny Crow's Garden", ListenURL: sectionURL(1)}, + {Title: "Johnny Crow's Party", ListenURL: sectionURL(2)}, + {Title: "Johnny Crow's New Garden", ListenURL: sectionURL(3)}, + }}, + work: metaops.LibrivoxWork{Title: testTitle, Authors: []string{testAuthor}, + Recordings: []metaops.LibrivoxRecording{{ID: testRec, LibrivoxID: testID, ArchiveID: testItem}}}, + } + x.fetch = &fakeFetcher{files: map[string]*remote{ + sectionURL(1): {body: mp3('1', 1000), etag: `"etag-1"`}, + sectionURL(2): {body: mp3('2', 2000), etag: `"etag-2"`}, + sectionURL(3): {body: mp3('3', 3000), etag: `"etag-3"`}, + coverURL: {body: jpeg}, + }} + x.srv = &fakeServer{} + x.libs = &fakeLibs{mount: x.mount, writable: true} + x.h = New(Options{Meta: x.meta, Fetcher: x.fetch, Libraries: x.libs, StateDir: x.stateDir, + Mount: func(id int64, _ string) (string, bool) { return x.libs.mountOf(id), true }, + Server: func() Server { return x.srv }, + Free: func(string) (int64, error) { return x.free, nil }, + Wait: func(ctx context.Context, _ time.Duration) error { return ctx.Err() }, + onProgress: func(p jobs.Progress) { + x.mu.Lock() + x.progress = append(x.progress, p) + x.mu.Unlock() + }}) + return x +} + +func (x *harness) job(resumed bool) jobs.Job { + lib := int64(3) + return jobs.Job{ID: 41, Kind: Kind, LibraryID: &lib, Attempt: 1, Resumed: resumed, + Params: []byte(fmt.Sprintf(`{"librivox_id":%d,"work_id":%q,"recording_id":%q,"library_id":3}`, testID, testWork, testRec))} +} + +func (x *harness) run(ctx context.Context, resumed bool) (any, error) { + return x.h.Run(ctx, x.job(resumed), jobs.LocalReporter()) +} + +func (x *harness) path(rel string) string { return filepath.Join(x.mount, filepath.FromSlash(rel)) } + +func (x *harness) exists(rel string) bool { + _, err := os.Lstat(x.path(rel)) + return err == nil +} + +func (x *harness) steps() []string { + x.mu.Lock() + defer x.mu.Unlock() + var out []string + for _, p := range x.progress { + if len(out) == 0 || out[len(out)-1] != p.Step { + out = append(out, p.Step) + } + } + return out +} + +// failureCode is the run's coded failure, "" for none. +func failureCode(t *testing.T, out any, err error) string { + t.Helper() + var f *Failure + if !errors.As(err, &f) { + return "" + } + if r, ok := out.(*Failure); !ok || r != f { + t.Fatalf("a failure's result isn't the failure: %#v", out) + } + return f.Code +} diff --git a/internal/librivoxjob/librivoxjob.go b/internal/librivoxjob/librivoxjob.go new file mode 100644 index 0000000..687b3e1 --- /dev/null +++ b/internal/librivoxjob/librivoxjob.go @@ -0,0 +1,844 @@ +// Package librivoxjob runs librivox.fetch (LIBRIVOX-PLAN.md L6): one LibriVox +// recording, read from the community metadata database's LibriVox-shaped feed, +// downloaded section by section from the Internet Archive into a staging +// folder in a books library, placed as Author/Title/NN - Section.mp3 with its +// cover.jpg and .audiosilo/librivox.json, then rescanned. +package librivoxjob + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path" + "regexp" + "slices" + "strings" + "time" + + "github.com/kodestar/audiosilo-bridge/internal/fsutil" + "github.com/kodestar/audiosilo-bridge/internal/importjob" + "github.com/kodestar/audiosilo-bridge/internal/jobs" + "github.com/kodestar/audiosilo-bridge/internal/link" + "github.com/kodestar/audiosilo-bridge/internal/mapping" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/netguard" + "github.com/kodestar/audiosilo-bridge/internal/transfer" +) + +// Kind is the job kind (its family, "librivox", is the Bridge feature the +// server's console and player gate Add from LibriVox on). +const Kind = "librivox.fetch" + +// The steps a fetch reports (progress.step), in order. +const ( + StepManifest = "manifest" + StepDownload = "download" + StepCover = "cover" + StepPlace = "place" + StepRescan = "rescan" +) + +// The codes a failed fetch reports (its result {code, message}); any other +// failure has none, only its error. +const ( + CodeNotWritable = "not_writable" // not a checked writable books mapping, or no valid path there + CodeExists = "exists" // the title's folder and its fallback are both taken + CodeDiskFull = "disk_full" // the library is short of room + CodeNotFound = "not_found" // meta has no such project (or it lists no files) + CodeArchiveUnavailable = "archive_unavailable" // a file isn't on archive.org, or archive.org can't be reached + CodeMetadataOff = "metadata_off" // the Bridge has no metadata base URL +) + +// Failure is a failed fetch's result and error: a code from the list above and +// a sentence for the console. It never holds a URL. +type Failure struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func (f *Failure) Error() string { return f.Message } + +func fail(code, format string, args ...any) *Failure { + return &Failure{Code: code, Message: fmt.Sprintf(format, args...)} +} + +// Params are librivox.fetch's params, built by the server: the LibriVox +// project, the work and recording the server resolved it to in meta, and the +// books library. Unknown fields are ignored (the server may add some). +type Params struct { + LibrivoxID int `json:"librivox_id"` + WorkID string `json:"work_id"` + RecordingID string `json:"recording_id"` + LibraryID int64 `json:"library_id"` +} + +// metaID is the shape of a meta work or recording id (a slug). +var metaID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$`) + +// ParseParams reads and checks the params. +func ParseParams(raw json.RawMessage) (Params, error) { + var p Params + if err := json.Unmarshal(raw, &p); err != nil { + return Params{}, fmt.Errorf("bad params: %w", err) + } + switch { + case p.LibrivoxID <= 0: + return Params{}, errors.New("bad params: librivox_id must be a LibriVox project id") + case !metaID.MatchString(p.WorkID) || !metaID.MatchString(p.RecordingID): + return Params{}, errors.New("bad params: work_id and recording_id must be meta ids") + case p.LibraryID <= 0: + return Params{}, errors.New("bad params: library_id is required") + } + return p, nil +} + +// Result is a done fetch: the book's path as the server indexed it, the +// project, and the files placed (the sections and the cover) with their bytes. +type Result struct { + LibraryID int64 `json:"library_id"` + Path string `json:"path"` + LibrivoxID int `json:"librivox_id"` + Files int `json:"files"` + Bytes int64 `json:"bytes"` +} + +// Marker is .audiosilo/librivox.json, the book's LibriVox identity in its own +// folder (decision 3; the server doesn't read it yet). +type Marker struct { + Version int `json:"version"` + LibrivoxID int `json:"librivox_id"` + WorkID string `json:"work_id"` + RecordingID string `json:"recording_id"` + ArchiveID string `json:"archive_id"` +} + +// MarkerVersion is librivox.json's version. +const MarkerVersion = 1 + +// markerRel is the marker inside the book's folder. +const markerRel = ".audiosilo/librivox.json" + +// Meta is the metadata client a fetch reads (*metaops.Client). +type Meta interface { + LibrivoxAudiobook(ctx context.Context, id int) (metaops.LibrivoxBook, error) + LibrivoxWork(ctx context.Context, workID string) (metaops.LibrivoxWork, error) +} + +// Fetcher is the guarded HTTP client the files come through (*netguard.Client). +type Fetcher interface { + Stream(ctx context.Context, in netguard.StreamRequest) (*netguard.StreamResponse, error) + Fetch(ctx context.Context, in netguard.Request) (*netguard.Response, error) +} + +// Server is the part of the server link a fetch calls (*link.Client). +type Server interface { + importjob.BookLister + Rescan(ctx context.Context, libraryID int64, path string) (string, error) +} + +// Libraries is the mapping registry a fetch checks (*mapping.Registry). +type Libraries interface { + Library(id int64) (link.Library, bool) + Status(id int64) (link.LibraryStatus, bool) +} + +// Options wire the handler. +type Options struct { + Meta Meta + Fetcher Fetcher + Server func() Server + Libraries Libraries + Mount mapping.MountFunc + // StateDir is <data>/librivox: each fetch's plan and partial downloads' + // validators (state). + StateDir string + Log *slog.Logger + + // Seams (tests): the free-space reader and the wait between attempts and + // between looks at the library's mapping. + Free func(dir string) (int64, error) + Wait func(ctx context.Context, d time.Duration) error + // onProgress sees every progress the job reports (tests). + onProgress func(jobs.Progress) +} + +// Handler runs librivox.fetch on the download lane (capacity 1, shared with +// audible.import and an ordinary podcast.fetch): one book at a time. +type Handler struct { + o Options +} + +// New returns the handler. +func New(o Options) *Handler { + if o.Log == nil { + o.Log = slog.New(slog.DiscardHandler) + } + if o.Free == nil { + o.Free = fsutil.FreeBytes + } + if o.Wait == nil { + o.Wait = netguard.Wait + } + return &Handler{o: o} +} + +// Kind is librivox.fetch. +func (h *Handler) Kind() string { return Kind } + +// Lane is the download lane. +func (h *Handler) Lane() string { return jobs.LaneDownload } + +// mappingPoll is how often a job waiting on its library's mapping looks again. +const mappingPoll = time.Second + +// Abandon removes the staging folder of a job the runner ended without +// running it (cancelled before it resumed). A lost claim keeps it: the server +// hands the job out again and the next claim resumes it. A placed book stays. +// The state is the job's library's alone, so a fetch of the same project +// into another library is never touched. +func (h *Handler) Abandon(ctx context.Context, job jobs.Job) { + if jobs.Lost(ctx) { + return + } + p, err := ParseParams(job.Params) + if err != nil { + return + } + st := h.loadState(p.LibraryID, p.LibrivoxID) + if st == nil || st.Placed { + return + } + if lib, ok := h.o.Libraries.Library(st.LibraryID); ok { + if mount, _ := h.o.Mount(lib.ID, lib.Root); mount != "" { + h.removeStaging(mount, p.LibrivoxID, st) + } + } + h.dropState(st.LibraryID, p.LibrivoxID) +} + +// Run fetches the recording: manifest, download, cover, place, rescan. +func (h *Handler) Run(ctx context.Context, job jobs.Job, rep *jobs.Reporter) (any, error) { + p, err := ParseParams(job.Params) + if err != nil { + return nil, err + } + if job.LibraryID != nil && *job.LibraryID != p.LibraryID { + return nil, errors.New("bad params: library_id isn't the job's library") + } + res, err := h.run(ctx, job, p, rep) + var f *Failure + if errors.As(err, &f) { + return f, f + } + if err != nil { + return nil, err + } + return res, nil +} + +// fetch is one run's working set. +type fetch struct { + p Params + lib link.Library + mount string + st *state + rep *jobs.Reporter +} + +// staging is the fetch's staging folder (library-relative). +func (f *fetch) staging() string { return f.st.staging(f.p.LibrivoxID) } + +// openRoot opens the library at its mount for the fetch's writes. +func (f *fetch) openRoot() (*os.Root, error) { + root, err := os.OpenRoot(f.mount) + if err != nil { + return nil, fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + } + return root, nil +} + +func (h *Handler) run(ctx context.Context, job jobs.Job, p Params, rep *jobs.Reporter) (res *Result, err error) { + var srv Server + if h.o.Server != nil { + srv = h.o.Server() + } + if srv == nil { + return nil, errors.New("the Bridge isn't paired with a server") + } + lib, mount, err := h.mapped(ctx, p.LibraryID, rep, StepManifest, !job.Resumed) + if err != nil { + return nil, err + } + f := &fetch{p: p, lib: lib, mount: mount, rep: rep, st: h.loadState(lib.ID, p.LibrivoxID)} + defer func() { + // A stop the next start resumes, and a lost claim the server hands out + // again, keep the staging folder and the state; a cancel or a failure + // before the book was placed removes both. A placed book stays, and its + // state with it when the rescan failed (the retry goes on from there). + var keep *keepStaging + if err == nil || f.st == nil || f.st.Placed || errors.As(err, &keep) { + return + } + if ctx.Err() != nil && (jobs.Resumes(ctx) || jobs.Lost(ctx)) { + return + } + h.removeStaging(f.mount, p.LibrivoxID, f.st) + h.dropState(lib.ID, p.LibrivoxID) + }() + + // An earlier run placed the book and then failed (the rescan): while its + // folder is still this project's, go on from the rescan. + if f.st != nil { + h.adopt(f) + } + if f.st != nil && f.st.Placed { + if h.holdsProject(mount, f.st.Dest, p.LibrivoxID) { + return h.finish(ctx, f, srv) + } + h.dropState(lib.ID, p.LibrivoxID) + f.st = nil + } + + h.report(rep, jobs.Progress{Step: StepManifest, Message: "Reading the recording from the metadata database"}) + book, work, err := h.manifest(ctx, p, rep) + if err != nil { + return nil, err + } + names := sectionNames(book.Sections) + archiveID := archiveIDOf(p, book, work) + + if f.st == nil { + if err := h.plan(ctx, f, srv, book, work); err != nil { + return nil, err + } + if f.st.Placed { + return h.finish(ctx, f, srv) // the folder is already this project's + } + } + if err := h.prepare(f); err != nil { + return nil, err + } + if err := h.download(ctx, f, book.Sections, names); err != nil { + return nil, err + } + // The mapping again before the cover's and the marker's writes. + if err := h.recheck(ctx, f, StepCover); err != nil { + return nil, err + } + h.cover(ctx, f, book) + if err := ctx.Err(); err != nil { + return nil, err + } + if err := h.writeMarker(f, archiveID); err != nil { + return nil, err + } + if err := h.place(ctx, f, names); err != nil { + return nil, err + } + return h.finish(ctx, f, srv) +} + +// manifest reads the project's manifest and its work from meta. A work meta +// doesn't have (another meta base than the server's, decision 5) is no +// failure: the project names the folder then. +// +// meta unavailable (a 5xx, a dropped connection) is never terminal: each read +// is tried metaAttempts times, and still unavailable the run fails with a +// keepStaging error, which keeps the staging folder and the state for the +// next attempt. Only meta's definite 404 (not_found) ends the fetch. +func (h *Handler) manifest(ctx context.Context, p Params, rep *jobs.Reporter) (metaops.LibrivoxBook, *metaops.LibrivoxWork, error) { + metadataOff := func() error { + return fail(CodeMetadataOff, "The Bridge has no metadata service set (metadata.base_url), so it can't read LibriVox recordings.") + } + if h.o.Meta == nil { + return metaops.LibrivoxBook{}, nil, metadataOff() + } + book, err := metaRead(ctx, h, rep, func() (metaops.LibrivoxBook, error) { return h.o.Meta.LibrivoxAudiobook(ctx, p.LibrivoxID) }) + switch { + case errors.Is(err, metaops.ErrDisabled): + return book, nil, metadataOff() + case errors.Is(err, metaops.ErrNotFound): + return book, nil, fail(CodeNotFound, "The metadata database has no LibriVox recording %d.", p.LibrivoxID) + case err != nil: + return book, nil, fmt.Errorf("reading LibriVox recording %d from the metadata database: %w", p.LibrivoxID, err) + case len(book.Sections) == 0: + return book, nil, fail(CodeNotFound, "LibriVox recording %d lists no files in the metadata database.", p.LibrivoxID) + } + for i, s := range book.Sections { + if !allowedRaw(s.ListenURL) { + return book, nil, fail(CodeArchiveUnavailable, "Section %d of LibriVox recording %d isn't a file on archive.org, the only place the Bridge downloads LibriVox from.", i+1, p.LibrivoxID) + } + } + w, err := metaRead(ctx, h, rep, func() (metaops.LibrivoxWork, error) { return h.o.Meta.LibrivoxWork(ctx, p.WorkID) }) + switch { + case errors.Is(err, metaops.ErrNotFound): + return book, nil, nil + case err != nil: + return book, nil, fmt.Errorf("reading the work of LibriVox recording %d from the metadata database: %w", p.LibrivoxID, err) + } + for _, r := range w.Recordings { + if r.ID == p.RecordingID && r.LibrivoxID > 0 && r.LibrivoxID != p.LibrivoxID { + return book, nil, fail(CodeNotFound, "The metadata database's recording %s is LibriVox recording %d now, not %d: search again.", + p.RecordingID, r.LibrivoxID, p.LibrivoxID) + } + } + return book, &w, nil +} + +// metaAttempts bounds one meta read while meta is unavailable; metaWait is +// the first wait between two, doubled each time (5 s, 10 s, 20 s). +const ( + metaAttempts = 4 + metaWait = 5 * time.Second +) + +// keepStaging is a run's failure that leaves the staging folder and the state +// for the next attempt (meta unavailable): nothing the fetch holds is wrong. +type keepStaging struct{ err error } + +func (k *keepStaging) Error() string { return k.err.Error() } +func (k *keepStaging) Unwrap() error { return k.err } + +// metaRead is read, tried again while meta is unavailable +// (metaops.ErrUnavailable); still unavailable after metaAttempts, the error is +// a keepStaging. Any other answer (a 404, metadata off, the context) is +// returned at once. +func metaRead[T any](ctx context.Context, h *Handler, rep *jobs.Reporter, read func() (T, error)) (T, error) { + wait := metaWait + for attempt := 1; ; attempt++ { + v, err := read() + if err == nil || !errors.Is(err, metaops.ErrUnavailable) || ctx.Err() != nil { + return v, err + } + if attempt == metaAttempts { + return v, &keepStaging{err: err} + } + h.report(rep, jobs.Progress{Step: StepManifest, Message: "The metadata database isn't answering: trying again"}) + if werr := h.o.Wait(ctx, wait); werr != nil { + return v, werr + } + wait *= 2 + } +} + +// archiveIDOf is the project's Internet Archive item: the recording's in the +// work, else the feed's url_iarchive. +func archiveIDOf(p Params, book metaops.LibrivoxBook, work *metaops.LibrivoxWork) string { + if work != nil { + for _, r := range work.Recordings { + if r.ID == p.RecordingID && r.ArchiveID != "" { + return r.ArchiveID + } + } + } + return book.ArchiveID() +} + +// plan picks the book's folder and records it (f.st): the title's folder under +// the author's, else once "<Title> (LibriVox <id>)", else exists. A folder +// that already holds this project (a placement whose state was lost) is the +// book: f.st comes back Placed. +func (h *Handler) plan(ctx context.Context, f *fetch, srv Server, book metaops.LibrivoxBook, work *metaops.LibrivoxWork) error { + books, err := importjob.AllBooks(ctx, srv, f.lib.ID) + if err != nil { + return fmt.Errorf("reading the library's books on the server: %w", err) + } + author, title := bookNames(f.p.LibrivoxID, book, work) + parent, dest := destination(author, title, books) + root, err := f.openRoot() + if err != nil { + return err + } + defer func() { _ = root.Close() }() + if parent != "" { + // The author folder as the library spells it (a case-insensitive disk). + spelled, err := transfer.OnDisk(transfer.RootFS(root), parent) + if err != nil { + return fail(CodeNotWritable, "The Bridge can't place the book under %s in the library: %v", parent, err) + } + parent = spelled + if err := h.parentFree(root, parent, books); err != nil { + return err + } + } + st := &state{LibraryID: f.lib.ID, Parent: parent} + dest = path.Join(parent, path.Base(dest)) // under the author folder as spelled on disk + fallback := fallbackName(dest, f.p.LibrivoxID) + for _, at := range []string{dest, fallback} { + if _, err := transfer.CleanRel(at); err != nil { + return fail(CodeNotWritable, "The book's folder %q isn't a valid path in the library.", at) + } + _, err := root.Lstat(at) + switch { + case errors.Is(err, fs.ErrNotExist): + st.Dest = at + case err != nil: + return fail(CodeNotWritable, "The Bridge can't look at %s in the library: %v", at, err) + case holdsProjectIn(root, at, f.p.LibrivoxID): + st.Dest, st.Placed = at, true + st.Count, st.Bytes = countFiles(root, at) + } + if st.Dest != "" { + break + } + } + if st.Dest == "" { + return fail(CodeExists, "%s and %s are both taken in the library: nothing was overwritten. Move or rename one of them, then add the recording again.", + dest, fallback) + } + if !st.Placed { + if _, err := root.Lstat(parent); parent != "" && errors.Is(err, fs.ErrNotExist) { + st.MadeParent = true + } + } + if err := h.saveState(f.p.LibrivoxID, st); err != nil { + return fmt.Errorf("recording the fetch's plan in the Bridge's data: %w", err) + } + f.st = st + return nil +} + +// parentFree refuses an author folder that is itself a book (one the server +// lists, or one holding audio): the book's folder in it would join that book. +func (h *Handler) parentFree(root *os.Root, parent string, books []link.LibraryBook) error { + for _, b := range books { + if b.Path == parent { + return fail(CodeExists, "%s is already the library's book %q, so the recording can't go in a folder inside it.", parent, b.Title) + } + } + entries, err := fs.ReadDir(transfer.RootFS(root), parent) + switch { + case errors.Is(err, fs.ErrNotExist): + return nil + case err != nil: + return fail(CodeNotWritable, "The Bridge can't read the folder %s in the library: %v", parent, err) + } + for _, e := range entries { + if !e.IsDir() && audioExts[strings.ToLower(path.Ext(e.Name()))] { + return fail(CodeExists, "%s already holds audio (%s), a book of its own, so the recording can't go in a folder inside it.", parent, e.Name()) + } + } + return nil +} + +// prepare makes the author folder and the staging folder (0755, through the +// mount's root). +func (h *Handler) prepare(f *fetch) error { + root, err := f.openRoot() + if err != nil { + return err + } + defer func() { _ = root.Close() }() + if f.st.Parent != "" { + if err := root.MkdirAll(f.st.Parent, 0o755); err != nil { + return writeFailure(err, f.st.Parent) + } + } + staging := f.staging() + if err := root.Mkdir(staging, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return writeFailure(err, staging) + } + if st, err := root.Lstat(staging); err != nil || !st.IsDir() { + return fail(CodeNotWritable, "The Bridge can't use %s in the library as its staging folder.", staging) + } + return nil +} + +// writeFailure is a failed write in the library as a fetch's failure: out of +// space is disk_full, a read-only filesystem or a refusal not_writable. +func writeFailure(err error, rel string) error { + switch { + case fsutil.IsNoSpace(err): + return fail(CodeDiskFull, "The library ran out of space writing %s.", rel) + case errors.Is(err, fs.ErrPermission), fsutil.IsReadOnly(err): + return fail(CodeNotWritable, "The Bridge can't write %s in the library: %v", rel, err) + } + return fmt.Errorf("writing %s in the library: %w", rel, err) +} + +// removeStaging removes the staging folder, and the author folder this fetch +// made once it is empty, through the mount's root (best effort). +func (h *Handler) removeStaging(mount string, id int, st *state) { + root, err := os.OpenRoot(mount) + if err != nil { + return + } + defer func() { _ = root.Close() }() + if err := root.RemoveAll(st.staging(id)); err != nil { + h.o.Log.Warn("librivox: removing a fetch's staging folder failed", "librivox_id", id, "path", st.staging(id), "err", err) + } + if st.MadeParent && st.Parent != "" { + _ = root.Remove(st.Parent) // only when empty + } +} + +// writeMarker writes .audiosilo/librivox.json into the staging folder. +func (h *Handler) writeMarker(f *fetch, archiveID string) error { + b, err := json.Marshal(Marker{Version: MarkerVersion, LibrivoxID: f.p.LibrivoxID, WorkID: f.p.WorkID, + RecordingID: f.p.RecordingID, ArchiveID: archiveID}) + if err != nil { + return err + } + root, err := f.openRoot() + if err != nil { + return err + } + defer func() { _ = root.Close() }() + dir := path.Join(f.staging(), path.Dir(markerRel)) + if err := root.Mkdir(dir, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return writeFailure(err, dir) + } + if err := fsutil.WriteRootAtomic(root, dir, path.Base(markerRel), b, 0o644, nil); err != nil { + return writeFailure(err, path.Join(f.staging(), markerRel)) + } + return nil +} + +// holdsProject reports whether the folder rel holds this project's +// librivox.json (a book an earlier run placed). +func (h *Handler) holdsProject(mount, rel string, id int) bool { + root, err := os.OpenRoot(mount) + if err != nil { + return false + } + defer func() { _ = root.Close() }() + return holdsProjectIn(root, rel, id) +} + +func holdsProjectIn(root *os.Root, rel string, id int) bool { + b, err := root.ReadFile(path.Join(rel, markerRel)) + if err != nil { + return false + } + var m Marker + return json.Unmarshal(b, &m) == nil && m.LibrivoxID == id +} + +// countFiles is how many regular files the book's folder holds outside +// .audiosilo/ (the sections and the cover), and their bytes. +func countFiles(root *os.Root, rel string) (n int, bytes int64) { + entries, err := fs.ReadDir(transfer.RootFS(root), rel) + if err != nil { + return 0, 0 + } + for _, e := range entries { + if !e.Type().IsRegular() || strings.HasPrefix(e.Name(), ".") { + continue + } + if info, err := e.Info(); err == nil { + n++ + bytes += info.Size() + } + } + return n, bytes +} + +// place renames the staging folder to the book's folder: one no-replace +// rename, after the mapping is checked again (not writable now, the job parks +// and waits with what it downloaded). A folder that turned up at the +// destination meanwhile moves the book to the fallback name once, else exists. +func (h *Handler) place(ctx context.Context, f *fetch, names []string) error { + h.report(f.rep, jobs.Progress{Step: StepPlace, Message: "Placing the book in the library"}) + if err := h.recheck(ctx, f, StepPlace); err != nil { + return err + } + root, err := f.openRoot() + if err != nil { + return err + } + defer func() { _ = root.Close() }() + if err := sweep(root, f.staging(), names); err != nil { + return err + } + n, bytes := countFiles(root, f.staging()) + primary := f.st.Dest + for _, dest := range candidates(primary, f.p.LibrivoxID) { + err = fsutil.RenameNoReplace(root, f.staging(), dest) + if errors.Is(err, fs.ErrExist) { + continue + } + if err != nil { + return writeFailure(err, dest) + } + f.st.Dest, f.st.Placed, f.st.Count, f.st.Bytes = dest, true, n, bytes + f.st.Files = nil + if serr := h.saveState(f.p.LibrivoxID, f.st); serr != nil { + h.o.Log.Warn("librivox: recording a placed book failed", "librivox_id", f.p.LibrivoxID, "err", serr) + } + return nil + } + return fail(CodeExists, "Something else is already at %s in the library: nothing was overwritten. Move or rename it, then add the recording again.", primary) +} + +// sweep leaves the staging folder holding this manifest's files only: its +// sections, the cover and .audiosilo/. Anything else there is an earlier +// run's (a partial, or a section fetched under a title the manifest has +// changed since) and goes, so it is never placed with the book; a section +// that isn't there fails the placement. An entry is the manifest's by its +// name, or by being the same file as the one found under that name: a mount +// may list a name in another form than it was given (NFD for NFC). The +// sections are looked for again after the removals, so a mount that tells +// neither way never has an incomplete book placed. +func sweep(root *os.Root, staging string, names []string) error { + sections := func() ([]fs.FileInfo, error) { + infos := make([]fs.FileInfo, 0, len(names)+2) + for i, name := range names { + st, err := root.Lstat(path.Join(staging, name)) + if err != nil || !st.Mode().IsRegular() { + return nil, fmt.Errorf("the staging folder doesn't hold section %d of the recording's %d", i+1, len(names)) + } + infos = append(infos, st) + } + return infos, nil + } + keep, err := sections() + if err != nil { + return err + } + named := map[string]bool{} + for _, name := range names { + named[name] = true + } + for _, name := range []string{coverName, path.Dir(markerRel)} { + named[name] = true + if st, err := root.Lstat(path.Join(staging, name)); err == nil { + keep = append(keep, st) + } + } + entries, err := fs.ReadDir(transfer.RootFS(root), staging) + if err != nil { + return fmt.Errorf("reading the staging folder: %w", err) + } + for _, e := range entries { + if named[e.Name()] { + continue + } + info, err := e.Info() + if err != nil { + continue // gone since the listing + } + if slices.ContainsFunc(keep, func(k fs.FileInfo) bool { return os.SameFile(k, info) }) { + continue + } + if err := root.RemoveAll(path.Join(staging, e.Name())); err != nil { + return writeFailure(err, path.Join(staging, e.Name())) + } + } + _, err = sections() + return err +} + +// adopt marks a fetch placed whose staging folder is gone and whose folder +// (or its fallback) holds this project: the rename happened and the state's +// write after it didn't land (a kill between the two, a full data folder). +// Without it the resumed run would download the book again and place it a +// second time under the fallback name. +func (h *Handler) adopt(f *fetch) { + if f.st.Placed { + return + } + root, err := f.openRoot() + if err != nil { + return + } + defer func() { _ = root.Close() }() + if _, err := root.Lstat(f.staging()); !errors.Is(err, fs.ErrNotExist) { + return + } + for _, at := range candidates(f.st.Dest, f.p.LibrivoxID) { + if holdsProjectIn(root, at, f.p.LibrivoxID) { + f.st.Dest, f.st.Placed, f.st.Files = at, true, nil + f.st.Count, f.st.Bytes = countFiles(root, at) + return + } + } +} + +// rescanAttempts bounds the rescan's tries (the book is placed either way). +const rescanAttempts = 3 + +// finish asks the server to index the placed book and makes the result; the +// state goes once it is made. +func (h *Handler) finish(ctx context.Context, f *fetch, srv Server) (*Result, error) { + h.report(f.rep, jobs.Progress{Step: StepRescan, Message: "Asking the server to index the book"}) + var bookPath string + var err error + for attempt := 1; attempt <= rescanAttempts; attempt++ { + if bookPath, err = srv.Rescan(ctx, f.lib.ID, f.st.Dest); err == nil { + break + } + if ctx.Err() != nil || attempt == rescanAttempts { + return nil, fmt.Errorf("the book is in the library at %s, but the server couldn't index it: %w", f.st.Dest, err) + } + if werr := h.o.Wait(ctx, time.Duration(attempt)*5*time.Second); werr != nil { + return nil, werr + } + } + if bookPath == "" { + bookPath = f.st.Dest + } + h.o.Log.Info("librivox: added", "librivox_id", f.p.LibrivoxID, "library", f.lib.ID, "path", bookPath, + "files", f.st.Count, "bytes", f.st.Bytes) + h.dropState(f.lib.ID, f.p.LibrivoxID) + return &Result{LibraryID: f.lib.ID, Path: bookPath, LibrivoxID: f.p.LibrivoxID, Files: f.st.Count, Bytes: f.st.Bytes}, nil +} + +// report records the job's progress. +func (h *Handler) report(rep *jobs.Reporter, pr jobs.Progress) { + rep.Progress(pr) + if h.o.onProgress != nil { + h.o.onProgress(pr) + } +} + +// recheck is mapped for a fetch under way: the library must still be a +// checked, writable mapping (not writable now, the job parks until it is), +// and at the mount the staging folder is in. A mapping moved to another mount +// fails the fetch: what it staged isn't there. +func (h *Handler) recheck(ctx context.Context, f *fetch, step string) error { + _, mount, err := h.mapped(ctx, f.lib.ID, f.rep, step, false) + if err != nil { + return err + } + if mount != f.mount { + return fail(CodeNotWritable, "Library %s moved to another folder on the Bridge during the fetch: add the recording again.", f.lib.Name) + } + return nil +} + +// mapped is the fetch's books library and its mount once the library is a +// checked, writable mapping there (audiblejob's rule): a job claimed before +// the check waits for it; a mapping that isn't writable fails a new job +// (strict) and parks a resumed one, or one past its download, not_writable +// until it comes back. +func (h *Handler) mapped(ctx context.Context, id int64, rep *jobs.Reporter, step string, strict bool) (link.Library, string, error) { + for { + lib, ok := h.o.Libraries.Library(id) + if !ok || lib.LibraryKind() != "books" { + return link.Library{}, "", fail(CodeNotWritable, "The server lists no books library %d for the Bridge.", id) + } + mount, _ := h.o.Mount(lib.ID, lib.Root) + st, checked := h.o.Libraries.Status(lib.ID) + why := fmt.Sprintf("Library %s must be mapped and writable on the Bridge: check its mapping on the Bridge's Status tab.", lib.Name) + switch { + case checked && st.MountPath == mount && st.Mapped && st.Writable: + rep.Unpark() + return lib, mount, nil + case !checked || st.MountPath != mount: + h.report(rep, jobs.Progress{Step: step, Message: "Waiting for the library's mapping check"}) + case strict: + return link.Library{}, "", fail(CodeNotWritable, "%s", why) + default: + h.report(rep, jobs.Progress{Step: step, Message: "Needs attention: " + why, + Extra: map[string]any{"park": map[string]any{"code": CodeNotWritable}}}) + rep.Park(CodeNotWritable, why) + } + if err := h.o.Wait(ctx, mappingPoll); err != nil { + return link.Library{}, "", err + } + } +} diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go new file mode 100644 index 0000000..a6c9922 --- /dev/null +++ b/internal/librivoxjob/librivoxjob_test.go @@ -0,0 +1,1068 @@ +package librivoxjob + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path" + "slices" + "strings" + "syscall" + "testing" + "time" + + "github.com/kodestar/audiosilo-bridge/internal/jobs" + "github.com/kodestar/audiosilo-bridge/internal/link" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/netguard" +) + +const bookDir = "L. Leslie Brooke/Johnny Crow's Garden" + +var stagingDir = "L. Leslie Brooke/" + stagingName(testID) + +func TestFetchPlacesBook(t *testing.T) { + x := newHarness(t) + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + res, ok := out.(*Result) + if !ok { + t.Fatalf("result %#v", out) + } + want := Result{LibraryID: 3, Path: bookDir, LibrivoxID: testID, Files: 4, Bytes: 1000 + 2000 + 3000 + int64(len(jpeg))} + if *res != want { + t.Fatalf("result %+v, want %+v", *res, want) + } + b, _ := json.Marshal(res) + if string(b) != `{"library_id":3,"path":"L. Leslie Brooke/Johnny Crow's Garden","librivox_id":192,"files":4,"bytes":6014}` { + t.Fatalf("result JSON %s", b) + } + for i, name := range []string{"01 - Johnny Crow's Garden.mp3", "02 - Johnny Crow's Party.mp3", "03 - Johnny Crow's New Garden.mp3"} { + got, err := os.ReadFile(x.path(path.Join(bookDir, name))) + if err != nil || !bytes.Equal(got, x.fetch.files[sectionURL(i+1)].body) { + t.Fatalf("%s: %v", name, err) + } + if st, _ := os.Stat(x.path(path.Join(bookDir, name))); st.Mode().Perm() != 0o644 { + t.Errorf("%s mode %v", name, st.Mode().Perm()) + } + } + if got, _ := os.ReadFile(x.path(bookDir + "/cover.jpg")); !bytes.Equal(got, jpeg) { + t.Fatalf("cover %q", got) + } + // librivox.json exactly as decision 3 has it. + got, err := os.ReadFile(x.path(bookDir + "/.audiosilo/librivox.json")) + if err != nil { + t.Fatal(err) + } + if string(got) != `{"version":1,"librivox_id":192,"work_id":"johnny-crows-garden-brooke","recording_id":"johnny-crows-garden-brooke-librivox-192","archive_id":"johnny_crow_garden_librivox"}` { + t.Fatalf("librivox.json %s", got) + } + if x.exists(stagingDir) { + t.Fatal("the staging folder is left") + } + if entries, _ := os.ReadDir(x.stateDir); len(entries) != 0 { + t.Fatalf("state left: %v", entries) + } + if !slices.Equal(x.srv.rescanned, []string{bookDir}) { + t.Fatalf("rescanned %v", x.srv.rescanned) + } + if s := x.steps(); !slices.Equal(s, []string{StepManifest, StepDownload, StepCover, StepPlace, StepRescan}) { + t.Fatalf("steps %v", s) + } + // The download's done/total count sections. + var last jobs.Progress + for _, p := range x.progress { + if p.Step == StepDownload { + last = p + if p.Total != 3 || p.Done > 3 { + t.Fatalf("download progress %+v", p) + } + } + } + if last.Done != 3 { + t.Fatalf("last download progress %+v", last) + } + // Every request went through the allowlist. + for _, r := range x.fetch.seen() { + if !allowedRaw(r.url) { + t.Fatalf("requested %s", r.url) + } + } +} + +func TestFetchServerAnswersAnotherPath(t *testing.T) { + x := newHarness(t) + x.srv.answer = "L. Leslie Brooke/Johnny Crow's Garden (indexed)" + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Path != x.srv.answer { + t.Fatalf("%#v %v", out, err) + } +} + +func TestFetchNoCover(t *testing.T) { + for name, set := range map[string]func(x *harness){ + "none stated": func(x *harness) { x.meta.book.CoverartJPG = nil }, + "not on archive": func(x *harness) { c := "https://librivox.org/cover.jpg"; x.meta.book.CoverartJPG = &c }, + "not a JPEG": func(x *harness) { x.fetch.files[coverURL].body = []byte("<html>") }, + "archive refuses": func(x *harness) { x.fetch.files[coverURL].status = 404 }, + } { + t.Run(name, func(t *testing.T) { + x := newHarness(t) + set(x) + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if x.exists(bookDir+"/cover.jpg") || out.(*Result).Files != 3 { + t.Fatalf("a cover was placed: %#v", out) + } + for _, r := range x.fetch.seen() { + if strings.Contains(r.url, "librivox.org") { + t.Fatal("librivox.org was requested") + } + } + }) + } +} + +// The host allowlist: a manifest file off archive.org, or over http, fails +// before anything is requested or written; a redirect off archive.org fails +// the job and leaves nothing. +func TestFetchHostAllowlist(t *testing.T) { + for name, listen := range map[string]string{ + "librivox.org": "https://librivox.org/x.mp3", + "http": "http://www.archive.org/download/x/1.mp3", + "lookalike": "https://archive.org.example/x.mp3", + "other port": "https://archive.org:8080/x.mp3", + "not a url": "::", + "empty": "", + "user and pass": "https://u:p@archive.org/x.mp3", + } { + t.Run(name, func(t *testing.T) { + x := newHarness(t) + x.meta.book.Sections[1].ListenURL = listen + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if len(x.fetch.seen()) != 0 { + t.Fatalf("requested %v", x.fetch.seen()) + } + if entries, _ := os.ReadDir(x.mount); len(entries) != 0 { + t.Fatalf("wrote %v", entries) + } + }) + } + t.Run("redirect off archive.org", func(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].redirect = "https://librivox.org/elsewhere.mp3" + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if strings.Contains(err.Error(), "librivox.org") || strings.Contains(err.Error(), "https://") { + t.Fatalf("a URL in the message: %v", err) + } + if x.exists(stagingDir) || x.exists("L. Leslie Brooke") || x.exists(bookDir) { + t.Fatal("left files behind") + } + }) + t.Run("redirect within archive.org", func(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].redirect = "https://ia800300.us.archive.org/1/items/x/2.mp3" + if _, err := x.run(context.Background(), false); err != nil { + t.Fatal(err) + } + }) +} + +// A stop during a section keeps the staging folder: the next run skips the +// finished sections and resumes the partial one with Range/If-Range. +func TestFetchResumesPartial(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) // a stop: Resumes + x.fetch.files[sectionURL(2)].cut = []int{700} + x.fetch.files[sectionURL(2)].onCut = stop + if _, err := x.run(ctx, false); !errors.Is(err, context.Canceled) { + t.Fatalf("first run: %v", err) + } + if !x.exists(stagingDir + "/01 - Johnny Crow's Garden.mp3") { + t.Fatal("the finished section went") + } + part, err := os.ReadFile(x.path(stagingDir + "/02 - Johnny Crow's Party.mp3.part")) + if err != nil || len(part) != 700 { + t.Fatalf("partial %d bytes, %v", len(part), err) + } + before := len(x.fetch.seen()) + + out, err := x.run(context.Background(), true) + if err != nil { + t.Fatal(err) + } + reqs := x.fetch.seen()[before:] + for _, r := range reqs { + if r.url == sectionURL(1) { + t.Fatal("a finished section was downloaded again") + } + } + if reqs[0].url != sectionURL(2) || reqs[0].offset != 700 || reqs[0].etag != `"etag-2"` { + t.Fatalf("the resume asked %+v", reqs[0]) + } + got, _ := os.ReadFile(x.path(bookDir + "/02 - Johnny Crow's Party.mp3")) + if !bytes.Equal(got, x.fetch.files[sectionURL(2)].body) { + t.Fatalf("resumed file %d bytes", len(got)) + } + if out.(*Result).Files != 4 { + t.Fatalf("%#v", out) + } +} + +// A partial without a strong ETag can't be continued: the section starts again. +func TestFetchRestartsWithoutStrongETag(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].etag = `W/"weak"` + x.fetch.files[sectionURL(2)].cut = []int{500} + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(x.path(bookDir + "/02 - Johnny Crow's Party.mp3")) + if !bytes.Equal(got, x.fetch.files[sectionURL(2)].body) || out.(*Result).Bytes != 6014 { + t.Fatalf("restarted file %d bytes", len(got)) + } +} + +// Dropped connections that add to the partial go on; three in a row that add +// nothing fail archive_unavailable and remove the staging folder. +func TestFetchDroppedConnections(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(3)].cut = []int{1000, 1000, 500} + if _, err := x.run(context.Background(), false); err != nil { + t.Fatal(err) + } + x = newHarness(t) + x.fetch.files[sectionURL(3)].cut = []int{0, 0, 0} + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) || x.exists("L. Leslie Brooke") { + t.Fatal("a failure left the staging folder") + } +} + +// A file archive.org doesn't have fails at once, and the staging folder goes +// (a terminal failure), the author folder this fetch made too. +func TestFetchMissingFileCleansUp(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(3)].status = 404 + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) || x.exists("L. Leslie Brooke") { + t.Fatal("left the staging or author folder") + } + if entries, _ := os.ReadDir(x.stateDir); len(entries) != 0 { + t.Fatalf("state left: %v", entries) + } + // An author folder that was there stays. + x = newHarness(t) + if err := os.MkdirAll(x.path("L. Leslie Brooke/Other"), 0o755); err != nil { + t.Fatal(err) + } + x.fetch.files[sectionURL(3)].status = 410 + if _, err := x.run(context.Background(), false); err == nil { + t.Fatal("no failure") + } + if x.exists(stagingDir) || !x.exists("L. Leslie Brooke/Other") { + t.Fatal("the cleanup took the wrong folders") + } +} + +// An admin's cancel removes the staging folder; a lost claim keeps it. +func TestFetchCancelAndLost(t *testing.T) { + x := newHarness(t) + ctx, cancel := jobs.CancelContext(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = cancel + if _, err := x.run(ctx, false); err == nil { + t.Fatal("a cancelled run succeeded") + } + if x.exists(stagingDir) || x.exists("L. Leslie Brooke") { + t.Fatal("a cancel left the staging folder") + } + if entries, _ := os.ReadDir(x.stateDir); len(entries) != 0 { + t.Fatalf("state left: %v", entries) + } + + x = newHarness(t) + ctx, lose := jobs.LoseContext(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = lose + if _, err := x.run(ctx, false); err == nil { + t.Fatal("a lost run succeeded") + } + if !x.exists(stagingDir + "/02 - Johnny Crow's Party.mp3.part") { + t.Fatal("a lost claim dropped the partial") + } +} + +func TestAbandon(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = stop + _, _ = x.run(ctx, false) + if !x.exists(stagingDir) { + t.Fatal("no staging folder to abandon") + } + x.h.Abandon(jobs.AbandonContext(context.Background(), true), x.job(true)) + if !x.exists(stagingDir) { + t.Fatal("a lost claim's Abandon removed the staging folder") + } + x.h.Abandon(jobs.AbandonContext(context.Background(), false), x.job(true)) + if x.exists(stagingDir) || x.exists("L. Leslie Brooke") { + t.Fatal("Abandon left the staging folder") + } + if entries, _ := os.ReadDir(x.stateDir); len(entries) != 0 { + t.Fatalf("state left: %v", entries) + } +} + +// The title's folder taken: once "<Title> (LibriVox <id>)", then exists with +// nothing written. +func TestFetchExistsFallback(t *testing.T) { + x := newHarness(t) + if err := os.MkdirAll(x.path(bookDir), 0o755); err != nil { + t.Fatal(err) + } + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if p := out.(*Result).Path; p != bookDir+" (LibriVox 192)" { + t.Fatalf("placed at %q", p) + } + if entries, _ := os.ReadDir(x.path(bookDir)); len(entries) != 0 { + t.Fatal("the existing folder was written to") + } + + x = newHarness(t) + for _, d := range []string{bookDir, bookDir + " (LibriVox 192)"} { + if err := os.MkdirAll(x.path(d), 0o755); err != nil { + t.Fatal(err) + } + } + out, err = x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeExists { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) || len(x.fetch.seen()) != 0 { + t.Fatal("an exists failure downloaded or staged") + } +} + +// A folder that turns up at the destination during the download: the rename +// never replaces it and takes the fallback name. +func TestFetchPlaceRace(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(3)].cut = []int{10} + x.fetch.files[sectionURL(3)].onCut = func() { _ = os.MkdirAll(x.path(bookDir+"/theirs"), 0o755) } + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if p := out.(*Result).Path; p != bookDir+" (LibriVox 192)" { + t.Fatalf("placed at %q", p) + } + if !x.exists(bookDir + "/theirs") { + t.Fatal("the other folder was replaced") + } +} + +// An author folder that is itself a book (listed, or holding audio) is never +// placed into. +func TestFetchParentIsABook(t *testing.T) { + x := newHarness(t) + x.srv.books = []link.LibraryBook{{Path: "L. Leslie Brooke", Title: "Loose", Authors: []string{"Someone Else"}}} + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeExists { + t.Fatalf("listed: code %q (%v)", code, err) + } + x = newHarness(t) + if err := os.MkdirAll(x.path("L. Leslie Brooke"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(x.path("L. Leslie Brooke/track.mp3"), []byte("a"), 0o644); err != nil { + t.Fatal(err) + } + out, err = x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeExists { + t.Fatalf("audio: code %q (%v)", code, err) + } +} + +// The author folder the library already uses is reused as it is spelled. +func TestFetchReusesAuthorFolder(t *testing.T) { + x := newHarness(t) + x.srv.books = []link.LibraryBook{{Path: "L.Leslie Brooke/Ring o' Roses", Title: "Ring o' Roses", Authors: []string{"L. Leslie Brooke"}}} + if err := os.MkdirAll(x.path("L.Leslie Brooke/Ring o' Roses"), 0o755); err != nil { + t.Fatal(err) + } + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Path != "L.Leslie Brooke/Johnny Crow's Garden" { + t.Fatalf("%#v %v", out, err) + } +} + +// A failed rescan keeps the placed book and its state: the next run goes on +// from the rescan, downloading nothing. +func TestFetchRescanFailureResumesAtRescan(t *testing.T) { + x := newHarness(t) + boom := errors.New("server busy") + x.srv.rescanErr = []error{boom, boom, boom} + if _, err := x.run(context.Background(), false); err == nil || !strings.Contains(err.Error(), "couldn't index it") { + t.Fatalf("first run: %v", err) + } + if !x.exists(bookDir + "/03 - Johnny Crow's New Garden.mp3") { + t.Fatal("a failed rescan removed the book") + } + before, reads := len(x.fetch.seen()), x.meta.reads + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if len(x.fetch.seen()) != before || x.meta.reads != reads { + t.Fatal("the retry read or downloaded again") + } + if r := out.(*Result); r.Path != bookDir || r.Files != 4 || r.Bytes != 6014 { + t.Fatalf("%+v", r) + } +} + +// A folder that already holds this project (its state lost) is the book: no +// download, only the rescan. +func TestFetchFolderAlreadyHoldsProject(t *testing.T) { + x := newHarness(t) + if err := os.MkdirAll(x.path(bookDir+"/.audiosilo"), 0o755); err != nil { + t.Fatal(err) + } + _ = os.WriteFile(x.path(bookDir+"/01 - a.mp3"), []byte("abc"), 0o644) + _ = os.WriteFile(x.path(bookDir+"/.audiosilo/librivox.json"), []byte(`{"version":1,"librivox_id":192}`), 0o644) + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if r := out.(*Result); r.Path != bookDir || r.Files != 1 || r.Bytes != 3 || len(x.fetch.seen()) != 0 { + t.Fatalf("%+v, %d requests", r, len(x.fetch.seen())) + } +} + +func TestFetchMetaFailures(t *testing.T) { + for _, tc := range []struct { + name string + set func(x *harness) + code string + }{ + {"metadata off", func(x *harness) { x.meta.bookErr = metaops.ErrDisabled }, CodeMetadataOff}, + {"no meta client", func(x *harness) { x.h.o.Meta = nil }, CodeMetadataOff}, + {"no such project", func(x *harness) { x.meta.book.ID = "1" }, CodeNotFound}, + {"no sections", func(x *harness) { x.meta.book.Sections = nil }, CodeNotFound}, + {"recording is another project", func(x *harness) { x.meta.work.Recordings[0].LibrivoxID = 5 }, CodeNotFound}, + {"meta down", func(x *harness) { x.meta.bookErr = metaops.ErrUnavailable }, ""}, + {"work unreadable", func(x *harness) { x.meta.workErr = metaops.ErrUnavailable }, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + x := newHarness(t) + tc.set(x) + out, err := x.run(context.Background(), false) + if err == nil { + t.Fatal("no failure") + } + if code := failureCode(t, out, err); code != tc.code { + t.Fatalf("code %q (%v)", code, err) + } + if entries, _ := os.ReadDir(x.mount); len(entries) != 0 { + t.Fatalf("wrote %v", entries) + } + }) + } + // A work meta doesn't have: the project names the folder. + x := newHarness(t) + x.meta.workErr = metaops.ErrNotFound + x.meta.book.Title = "Johnny Crow's Garden (version 2)" + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Path != "L. Leslie Brooke/Johnny Crow's Garden (version 2)" { + t.Fatalf("%#v %v", out, err) + } + b, _ := os.ReadFile(x.path("L. Leslie Brooke/Johnny Crow's Garden (version 2)/.audiosilo/librivox.json")) + var m Marker + if json.Unmarshal(b, &m) != nil || m.ArchiveID != testItem { + t.Fatalf("the archive id from url_iarchive: %s", b) + } +} + +func TestFetchNotWritable(t *testing.T) { + x := newHarness(t) + x.libs.writable = false + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeNotWritable { + t.Fatalf("code %q (%v)", code, err) + } + x = newHarness(t) + x.libs.kind = "podcasts" + out, err = x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeNotWritable { + t.Fatalf("podcasts: code %q (%v)", code, err) + } +} + +func TestFetchDiskFull(t *testing.T) { + x := newHarness(t) + x.free = 100 + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeDiskFull { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) { + t.Fatal("disk_full left the staging folder") + } +} + +func TestWriteFailureCodes(t *testing.T) { + var f *Failure + if !errors.As(writeFailure(syscall.ENOSPC, "x"), &f) || f.Code != CodeDiskFull { + t.Fatalf("ENOSPC: %v", f) + } + if !errors.As(writeFailure(os.ErrPermission, "x"), &f) || f.Code != CodeNotWritable { + t.Fatalf("permission: %v", f) + } + if errors.As(writeFailure(errors.New("other"), "x"), &f) { + t.Fatal("an unknown write error was coded") + } +} + +func TestParseParams(t *testing.T) { + good := `{"librivox_id":192,"work_id":"w","recording_id":"r","library_id":3,"extra":true}` + if p, err := ParseParams([]byte(good)); err != nil || p.LibrivoxID != 192 || p.LibraryID != 3 { + t.Fatalf("%+v %v", p, err) + } + for _, bad := range []string{ + `{`, `{"librivox_id":0,"work_id":"w","recording_id":"r","library_id":3}`, + `{"librivox_id":192,"work_id":"","recording_id":"r","library_id":3}`, + `{"librivox_id":192,"work_id":"../x","recording_id":"r","library_id":3}`, + `{"librivox_id":192,"work_id":"w","recording_id":"r/s","library_id":3}`, + `{"librivox_id":192,"work_id":"w","recording_id":"r","library_id":0}`, + } { + if _, err := ParseParams([]byte(bad)); err == nil { + t.Errorf("accepted %s", bad) + } + } + x := newHarness(t) + other := int64(4) + job := x.job(false) + job.LibraryID = &other + if _, err := x.h.Run(context.Background(), job, jobs.LocalReporter()); err == nil { + t.Fatal("a job for another library ran") + } +} + +// A state file whose paths aren't clean library paths is dropped, never +// followed (it decides what the cleanup removes). +func TestStateValid(t *testing.T) { + for _, tc := range []struct { + st state + ok bool + }{ + {state{LibraryID: 3, Parent: "A", Dest: "A/T"}, true}, + {state{LibraryID: 3, Parent: "", Dest: "T"}, true}, + {state{LibraryID: 3, Parent: "..", Dest: "../T"}, false}, + {state{LibraryID: 3, Parent: "A", Dest: "A/../T"}, false}, + {state{LibraryID: 3, Parent: "A", Dest: "B/T"}, false}, + {state{LibraryID: 0, Parent: "A", Dest: "A/T"}, false}, + } { + if got := tc.st.valid(); got != tc.ok { + t.Errorf("%+v valid = %v", tc.st, got) + } + } + x := newHarness(t) + _ = os.MkdirAll(x.stateDir, 0o700) + _ = os.WriteFile(x.h.statePath(3, 192), []byte(`{"library_id":3,"parent":"..","dest":"../T"}`), 0o600) + if x.h.loadState(3, 192) != nil { + t.Fatal("an invalid state loaded") + } + if _, err := os.Stat(x.h.statePath(3, 192)); err == nil { + t.Fatal("an invalid state stayed") + } + // A state file naming another library than its own is never followed. + _ = os.WriteFile(x.h.statePath(3, 192), []byte(`{"library_id":4,"parent":"A","dest":"A/T"}`), 0o600) + if x.h.loadState(3, 192) != nil { + t.Fatal("another library's state loaded") + } +} + +// A resumed job whose library isn't writable now parks not_writable and goes +// on once the mapping is back. +func TestFetchResumedParksForMapping(t *testing.T) { + x := newHarness(t) + x.libs.writable = false + waits := 0 + x.h.o.Wait = func(context.Context, time.Duration) error { + waits++ + x.libs.writable = true + return nil + } + out, err := x.run(context.Background(), true) + if err != nil || out.(*Result).Path != bookDir { + t.Fatalf("%#v %v", out, err) + } + parked := false + for _, p := range x.progress { + if park, ok := p.Extra["park"].(map[string]any); ok && park["code"] == CodeNotWritable { + parked = true + } + } + if !parked || waits == 0 { + t.Fatal("the resumed job didn't park") + } + if x.h.Kind() != "librivox.fetch" || x.h.Lane() != jobs.LaneDownload { + t.Fatal("kind or lane") + } +} + +// archive.org's 429 and 5xx are tried again; a 206 the Bridge can't prove +// continues its partial (a 416) starts the file again. +func TestFetchTransientStatuses(t *testing.T) { + x := newHarness(t) + flaky := &flakyFetcher{fakeFetcher: x.fetch, statuses: map[string][]int{ + sectionURL(1): {429, 503}, + sectionURL(2): {416}, + }} + x.h.o.Fetcher = flaky + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Bytes != 6014 { + t.Fatalf("%#v %v", out, err) + } + x = newHarness(t) + x.h.o.Fetcher = &flakyFetcher{fakeFetcher: x.fetch, statuses: map[string][]int{sectionURL(1): {503, 503, 503}}} + out, err = x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } +} + +// flakyFetcher answers a URL's queued statuses before its file. +type flakyFetcher struct { + *fakeFetcher + statuses map[string][]int + // html: that many 200 answers of an HTML page before the file. + html map[string]int +} + +func (f *flakyFetcher) Stream(ctx context.Context, in netguard.StreamRequest) (*netguard.StreamResponse, error) { + if f.html[in.URL] > 0 { + f.html[in.URL]-- + page := "<html>not found</html>" + return &netguard.StreamResponse{Status: 200, Body: io.NopCloser(strings.NewReader(page)), ContentLength: int64(len(page)), + ContentType: "text/html"}, nil + } + if q := f.statuses[in.URL]; len(q) > 0 { + f.statuses[in.URL] = q[1:] + return &netguard.StreamResponse{Status: q[0], Body: io.NopCloser(strings.NewReader("")), RetryAfterSeconds: 1}, nil + } + return f.fakeFetcher.Stream(ctx, in) +} + +// Without a strong ETag every attempt starts the file again: cut attempts +// that never get past what the partial held count as adding nothing, so the +// section fails instead of looping. +func TestFetchWeakETagCutsFail(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].etag = `W/"weak"` + x.fetch.files[sectionURL(2)].cut = []int{500, 500, 500, 500} + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } +} + +// A mapping that goes away during the download parks the job with what it +// fetched; it goes on once the mapping is back. +func TestFetchParksMidDownload(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = func() { x.libs.writable = false } + waits := 0 + x.h.o.Wait = func(context.Context, time.Duration) error { + // The first wait is the cut section's retry; the next is the park's. + if waits++; waits > 1 { + x.libs.writable = true + } + return nil + } + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Bytes != 6014 { + t.Fatalf("%#v %v", out, err) + } + parked := false + for _, p := range x.progress { + if park, ok := p.Extra["park"].(map[string]any); ok && park["code"] == CodeNotWritable && p.Step == StepDownload { + parked = true + } + } + if !parked { + t.Fatal("the job didn't park mid-download") + } +} + +// An empty body is never a section, whatever length the response stated: it +// is tried again and then fails, rather than being placed as a finished file. +func TestFetchEmptyBodyFails(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].body = nil + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(bookDir) || x.exists(stagingDir) { + t.Fatal("an empty section was placed or left staged") + } +} + +// Restarts from zero whose cuts only ever pass the LAST cut, never the most +// the partial has held, add nothing: the section fails instead of going on +// for as long as the cuts alternate. +func TestFetchRestartsNeverPassingTheirBestFail(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].etag = `W/"weak"` + x.fetch.files[sectionURL(2)].cut = []int{1000, 500, 600, 500, 600} + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if left := len(x.fetch.files[sectionURL(2)].cut); left != 1 { + t.Fatalf("%d cuts left: three attempts in a row that add nothing end the section", left) + } +} + +// A body from its start drops the old partial before its validators are +// recorded: a stop between the two can't leave the old bytes under the new +// ETag, which would prove a later 206 continues them. +func TestFetchRestartDropsPartialBeforeValidators(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{700} + x.fetch.files[sectionURL(2)].onCut = stop + if _, err := x.run(ctx, false); !errors.Is(err, context.Canceled) { + t.Fatalf("first run: %v", err) + } + part := stagingDir + "/02 - Johnny Crow's Party.mp3.part" + if !x.exists(part) { + t.Fatal("no partial to restart") + } + // The file changed on archive.org: the resume's If-Range gets the whole + // new file under a new ETag. + x.fetch.files[sectionURL(2)] = &remote{body: mp3('n', 2000), etag: `"etag-2b"`} + checked := false + x.h.o.Free = func(string) (int64, error) { + if !checked { // the restarted section's attempt, before its state is saved + checked = true + if x.exists(part) { + t.Error("the old partial was still there when the new validators were recorded") + } + } + return x.free, nil + } + if _, err := x.run(context.Background(), true); err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(x.path(bookDir + "/02 - Johnny Crow's Party.mp3")) + if !checked || !bytes.Equal(got, x.fetch.files[sectionURL(2)].body) { + t.Fatalf("restarted file %d bytes, checked %v", len(got), checked) + } +} + +// What an earlier run staged under names the manifest no longer has (a +// partial, a section whose title changed) is never placed with the book. +func TestFetchPlacesOnlyTheManifestsFiles(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{700} + x.fetch.files[sectionURL(2)].onCut = stop + if _, err := x.run(ctx, false); !errors.Is(err, context.Canceled) { + t.Fatalf("first run: %v", err) + } + x.meta.book.Sections[0].Title = "The Garden" + x.meta.book.Sections[1].Title = "The Party" + out, err := x.run(context.Background(), true) + if err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(x.path(bookDir)) + if err != nil { + t.Fatal(err) + } + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + want := []string{".audiosilo", "01 - The Garden.mp3", "02 - The Party.mp3", "03 - Johnny Crow's New Garden.mp3", "cover.jpg"} + if !slices.Equal(names, want) || out.(*Result).Files != 4 || out.(*Result).Bytes != 6014 { + t.Fatalf("placed %v, %+v", names, out) + } +} + +// A mapping moved to another mount during the fetch fails it (what it staged +// isn't there) and removes the staging folder from the mount it was in. +func TestFetchMountMovedFails(t *testing.T) { + x := newHarness(t) + old, moved := x.mount, t.TempDir() + x.fetch.files[sectionURL(1)].cut = []int{100} + x.fetch.files[sectionURL(1)].onCut = func() { x.mount, x.libs.mount = moved, moved } + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeNotWritable { + t.Fatalf("code %q (%v)", code, err) + } + for _, mount := range []string{old, moved} { + if entries, _ := os.ReadDir(mount); len(entries) != 0 { + t.Fatalf("%s holds %v after the failure", mount, entries) + } + } +} + +// A placement whose state never said so (a kill between the rename and the +// state's write) is found by the book's marker: the next run rescans it and +// doesn't download the book again under the fallback name. +func TestFetchAdoptsUnrecordedPlacement(t *testing.T) { + x := newHarness(t) + boom := errors.New("server busy") + x.srv.rescanErr = []error{boom, boom, boom} + if _, err := x.run(context.Background(), false); err == nil { + t.Fatal("the first run's rescan didn't fail") + } + st := x.h.loadState(3, testID) + if st == nil || !st.Placed { + t.Fatalf("state %+v", st) + } + st.Placed, st.Count, st.Bytes = false, 0, 0 + if err := x.h.saveState(testID, st); err != nil { + t.Fatal(err) + } + before := len(x.fetch.seen()) + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if r := out.(*Result); r.Path != bookDir || r.Files != 4 || r.Bytes != 6014 || len(x.fetch.seen()) != before { + t.Fatalf("%+v, %d new requests", r, len(x.fetch.seen())-before) + } + if x.exists(fallbackName(bookDir, testID)) || x.exists(stagingDir) { + t.Fatal("the book was fetched a second time") + } +} + +// A padded URL is refused as it stands: the check and the request see one +// spelling. +func TestFetchPaddedURLRefused(t *testing.T) { + for _, raw := range []string{" " + sectionURL(1), sectionURL(1) + " ", sectionURL(1) + "\n"} { + if allowedRaw(raw) { + t.Errorf("allowedRaw(%q)", raw) + } + } + if !allowedRaw(sectionURL(1)) { + t.Error("the URL itself refused") + } + x := newHarness(t) + x.meta.book.Sections[1].ListenURL += " " + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable || len(x.fetch.seen()) != 0 || x.exists("L. Leslie Brooke") { + t.Fatalf("code %q (%v), %d requests", code, err, len(x.fetch.seen())) + } +} + +// meta unavailable for a while is tried again inside the run, for the +// manifest and the work alike. +func TestFetchMetaUnavailableRetried(t *testing.T) { + x := newHarness(t) + u := metaops.ErrUnavailable + x.meta.bookErrs = []error{u, u} + x.meta.workErrs = []error{u} + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Path != bookDir { + t.Fatalf("%#v %v", out, err) + } + if x.meta.reads != 3 { + t.Fatalf("%d manifest reads", x.meta.reads) + } +} + +// meta unavailable past the retries on a resumed run fails without a code and +// keeps the staging folder and the state: the next attempt resumes, never +// fetching a finished section again. A definite 404 still ends the fetch. +func TestFetchMetaUnavailableKeepsStaging(t *testing.T) { + for _, which := range []string{"manifest", "work"} { + t.Run(which, func(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = stop + _, _ = x.run(ctx, false) + if !x.exists(stagingDir + "/01 - Johnny Crow's Garden.mp3") { + t.Fatal("no staged section") + } + u := metaops.ErrUnavailable + if which == "manifest" { + x.meta.bookErr = u + } else { + x.meta.workErr = u + } + out, err := x.run(context.Background(), true) + if err == nil || !errors.Is(err, metaops.ErrUnavailable) { + t.Fatalf("resumed run: %v", err) + } + if code := failureCode(t, out, err); code != "" { + t.Fatalf("meta unavailable was coded %q", code) + } + if !x.exists(stagingDir+"/01 - Johnny Crow's Garden.mp3") || x.h.loadState(3, testID) == nil { + t.Fatal("meta unavailable removed the staging folder or the state") + } + x.meta.bookErr, x.meta.workErr = nil, nil + before := len(x.fetch.seen()) + if _, err := x.run(context.Background(), true); err != nil { + t.Fatal(err) + } + for _, r := range x.fetch.seen()[before:] { + if r.url == sectionURL(1) { + t.Fatal("a finished section was fetched again") + } + } + }) + } + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = stop + _, _ = x.run(ctx, false) + x.meta.bookErr = metaops.ErrNotFound + out, err := x.run(context.Background(), true) + if code := failureCode(t, out, err); code != CodeNotFound { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) || x.h.loadState(3, testID) != nil { + t.Fatal("not_found kept the staging folder") + } +} + +// A section that isn't audio (a page sent with a 200, or bytes that are +// neither an ID3 tag nor an MPEG frame) is never kept: it is tried again, +// then archive_unavailable, and nothing is placed. +func TestFetchRefusesNonAudio(t *testing.T) { + for name, set := range map[string]func(r *remote){ + "html type": func(r *remote) { r.ctype = "text/html; charset=utf-8" }, + "json type": func(r *remote) { r.ctype = "application/json" }, + "html bytes": func(r *remote) { r.body = append([]byte("<!DOCTYPE html>"), r.body[15:]...) }, + "text bytes": func(r *remote) { r.body = bytes.Repeat([]byte("x"), len(r.body)) }, + } { + t.Run(name, func(t *testing.T) { + x := newHarness(t) + set(x.fetch.files[sectionURL(2)]) + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) || x.exists(bookDir) { + t.Fatal("a non-audio section was kept or placed") + } + }) + } + // Once: tried again, and the real file then is kept. + x := newHarness(t) + x.h.o.Fetcher = &flakyFetcher{fakeFetcher: x.fetch, html: map[string]int{sectionURL(2): 1}} + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Bytes != 6014 { + t.Fatalf("%#v %v", out, err) + } +} + +func TestAudioChecks(t *testing.T) { + for ct, want := range map[string]bool{ + "": true, "audio/mpeg": true, "audio/ogg; codecs=vorbis": true, "application/octet-stream": true, + "application/ogg": true, "text/html": false, "text/plain; charset=utf-8": false, "application/json": false, + "image/jpeg": false, ";;bad": false, "audio/mpeg; charset": true, "text/html; charset": false, + } { + if got := audioType(ct); got != want { + t.Errorf("audioType(%q) = %v", ct, got) + } + } + for head, want := range map[string]bool{ + "ID3\x04": true, "\xFF\xFB\x90": true, "\xFF\xF3": true, "OggS": true, "fLaC": true, "\x00\x00\x00\x20ftypM4A ": true, + "<html>": false, "": false, "\xFF\x00": false, "x": false, + "\x00\x00\x00\xFF\xFB\x90": true, "\x00\x00ID3\x03": true, "\x00\x00\x00\x00": false, "\x00\x00<!DO": false, + } { + if got := isAudioHead([]byte(head)); got != want { + t.Errorf("isAudioHead(%q) = %v", head, got) + } + } +} + +// Two libraries fetching one project keep their own state and staging: a run +// into one never discards the other's, and Abandon of one leaves the other. +func TestFetchTwoLibrariesIndependent(t *testing.T) { + x := newHarness(t) + other := t.TempDir() + x.libs.second = other + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = stop + _, _ = x.run(ctx, false) // library 3 stops mid-download + if !x.exists(stagingDir) { + t.Fatal("no staging in library 3") + } + // Library 4 fetches the whole project. + job4 := x.job(false) + lib4 := int64(4) + job4.LibraryID = &lib4 + job4.Params = bytes.Replace(job4.Params, []byte(`"library_id":3`), []byte(`"library_id":4`), 1) + out, err := x.h.Run(context.Background(), job4, jobs.LocalReporter()) + if err != nil || out.(*Result).LibraryID != 4 { + t.Fatalf("library 4: %#v %v", out, err) + } + if _, err := os.Stat(path.Join(other, bookDir, "01 - Johnny Crow's Garden.mp3")); err != nil { + t.Fatal("library 4's book isn't there") + } + if !x.exists(stagingDir+"/01 - Johnny Crow's Garden.mp3") || x.h.loadState(3, testID) == nil { + t.Fatal("library 4's fetch discarded library 3's staging") + } + // Library 4 stops mid-download too; Abandon of library 3's job leaves it. + x.fetch.files[sectionURL(2)].cut = []int{100} + ctx, stop = context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].onCut = stop + if err := os.RemoveAll(path.Join(other, bookDir)); err != nil { + t.Fatal(err) + } + _, _ = x.h.Run(ctx, job4, jobs.LocalReporter()) + staging4 := path.Join(other, stagingDir) + if _, err := os.Stat(staging4); err != nil { + t.Fatal("no staging in library 4") + } + x.h.Abandon(jobs.AbandonContext(context.Background(), false), x.job(true)) + if x.exists(stagingDir) { + t.Fatal("Abandon left library 3's staging") + } + if _, err := os.Stat(staging4); err != nil || x.h.loadState(4, testID) == nil { + t.Fatal("Abandon of library 3's job removed library 4's fetch") + } + // Library 3 resumes from scratch independently and finishes. + if _, err := x.run(context.Background(), false); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(staging4); err != nil { + t.Fatal("library 3's run removed library 4's staging") + } +} diff --git a/internal/librivoxjob/names.go b/internal/librivoxjob/names.go new file mode 100644 index 0000000..6ccdb86 --- /dev/null +++ b/internal/librivoxjob/names.go @@ -0,0 +1,202 @@ +package librivoxjob + +import ( + "fmt" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "github.com/kodestar/audiosilo-bridge/internal/importjob" + "github.com/kodestar/audiosilo-bridge/internal/link" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/placement" + "github.com/kodestar/audiosilo-server/pkg/match" +) + +// maxName bounds a book folder's or a section's own name (bytes, before the +// "NN - " prefix, the extension, ".part" or the " (LibriVox <id>)" suffix), so +// every name stays inside the 255 bytes a filesystem allows. +const maxName = 180 + +// variousAfter: a work with more authors than this is placed under Various. +const variousAfter = 3 + +// Various is the author folder of a work with more than variousAfter authors. +const Various = "Various" + +// segment is s as one path segment: control characters and runs of spaces as +// one space, placement's Sanitize (no "/", "\\" or NUL, ":" as " -", no +// leading or trailing dot or space), cut to maxName bytes on a rune boundary +// and trimmed again. "" when nothing is left ("..", " "). +func segment(s string) string { + s = strings.Join(strings.FieldsFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }), " ") + s = placement.Sanitize(s) + if len(s) > maxName { + cut := maxName + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + s = placement.Sanitize(s[:cut]) + } + return s +} + +// bookNames is the book's author and title as placement takes them: the title +// is the work's, or the project's own when the work has more than one LibriVox +// recording (each its own folder: "Metamorphosis (version 2)"), either when the +// other is empty, else "LibriVox <id>"; the author is the first credit, Various +// over three. work is nil when meta has no such work (the project's own then). +func bookNames(id int, book metaops.LibrivoxBook, work *metaops.LibrivoxWork) (author, title string) { + var authors []string + workTitle := "" + if work != nil { + authors, workTitle = work.Authors, work.Title + } + if len(authors) == 0 { + for _, a := range book.Authors { + if name := a.Name(); name != "" { + authors = append(authors, name) + } + } + } + switch { + case len(authors) > variousAfter: + author = Various + case len(authors) > 0: + author = authors[0] + } + first, second := workTitle, book.Title + if work != nil && work.LibrivoxCount() > 1 { + first, second = book.Title, workTitle + } + for _, t := range []string{first, second, "LibriVox " + strconv.Itoa(id)} { + if title = segment(t); title != "" { + break + } + } + return segment(author), title +} + +// destination is where the book's folder goes: placement's standalone rule (an +// existing author folder of the library reused as spelled, else the sanitized +// author, then the title), library-relative, with its parent folder ("" for +// the library's root: a book with no author). The title goes verbatim, never +// through match.CleanTitle (importjob.Planner's), which would drop the very +// "(version 2)" or "Vol. 001" that tells two LibriVox projects of one work apart. +func destination(author, title string, books []link.LibraryBook) (parent, dest string) { + folder := "" + if author != "" { + folder = importjob.AuthorFolders(books)[match.Normalize(author)] + } + sug := placement.Suggest(placement.Input{Author: author, Title: title, TitleShort: title, AuthorFolder: folder}) + dest = strings.Trim(sug.RelPath, "/") + parent = path.Dir(dest) + if parent == "." { + parent = "" + } + return parent, dest +} + +// fallbackName is the folder name tried once when the title's is taken. +func fallbackName(title string, id int) string { + return fmt.Sprintf("%s (LibriVox %d)", title, id) +} + +// candidates are the folders a fetch planned for dest may place the book at: +// dest, then once its fallback name, unless dest is the fallback already. +func candidates(dest string, id int) []string { + if strings.HasSuffix(dest, fallbackName("", id)) { + return []string{dest} + } + return []string{dest, fallbackName(dest, id)} +} + +// audioExts are the extensions a section file keeps from its URL; anything +// else is named .mp3 (LibriVox publishes its sections as MP3). +var audioExts = map[string]bool{".mp3": true, ".ogg": true, ".m4a": true, ".m4b": true, ".flac": true, ".opus": true} + +// sectionNames are the sections' file names: "NN - <title>.<ext>", NN the +// 1-based position (never LibriVox's section_number, which starts at 0 on some +// projects), zero-padded to the count's width and at least two digits, and a +// section with no title left "Section N". +func sectionNames(sections []metaops.LibrivoxSection) []string { + width := max(2, len(strconv.Itoa(len(sections)))) + out := make([]string, len(sections)) + for i, s := range sections { + n := i + 1 + name := segment(dropOwnNumber(s.Title, s.SectionNumber, n)) + if name == "" { + name = "Section " + strconv.Itoa(n) + } + out[i] = fmt.Sprintf("%0*d - %s%s", width, n, name, sectionExt(s.ListenURL)) + } + return out +} + +// ownNumber is a section title that opens with a number and a separator +// ("1 - ", "1. ", "1: ", "1) "), or is a number alone (a separator after it +// or not). "1.5 Interlude" is no match: the separator needs a space after it. +var ownNumber = regexp.MustCompile(`^\s*(\d+)(?:\s*[-.:)](?:\s+(.*)|\s*)|\s*)$`) + +// dropOwnNumber is a section's title without the number it opens with when +// that number is the section's own (its section_number, or its 1-based +// position pos): "1 - Chapter 01 through Chapter 06" is section 1's +// "Chapter 01 through Chapter 06", since its file is named "01 - " already. +// A title that is only its own number comes back "" (the caller names it +// "Section N"); any other number ("1984", "1984 - Part One" as section 1) is +// the title's own and stays. +func dropOwnNumber(title, sectionNumber string, pos int) string { + m := ownNumber.FindStringSubmatch(title) + if m == nil { + return title + } + num, err := strconv.Atoi(m[1]) + if err != nil { + return title + } + if sn, err := strconv.Atoi(strings.TrimSpace(sectionNumber)); (err != nil || num != sn) && num != pos { + return title + } + return m[2] +} + +// sectionExt is the file's extension from its URL's path (lower case), .mp3 +// when it names no audio one. +func sectionExt(raw string) string { + if u, err := url.Parse(raw); err == nil { + if ext := strings.ToLower(path.Ext(u.Path)); audioExts[ext] { + return ext + } + } + return ".mp3" +} + +// AllowedURL reports whether a fetch may request u (the first URL and every +// redirect's, netguard checking the addresses beside it): https on the default +// port, no credentials, and archive.org or a host under it. Never librivox.org +// (LIBRIVOX-PLAN.md L6 decision 7): the Bridge reads LibriVox through meta and +// its files from the Internet Archive only. +func AllowedURL(u *url.URL) bool { + if u == nil || u.Scheme != "https" || u.User != nil || u.Opaque != "" { + return false + } + if p := u.Port(); p != "" && p != "443" { + return false + } + host := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") + return host == "archive.org" || (strings.HasSuffix(host, ".archive.org") && len(host) > len(".archive.org")) +} + +// allowedRaw is AllowedURL over a URL string, as it is requested: a padded +// one is refused here rather than checked trimmed and then sent as it is. +func allowedRaw(raw string) bool { + if raw != strings.TrimSpace(raw) { + return false + } + u, err := url.Parse(raw) + return err == nil && AllowedURL(u) +} diff --git a/internal/librivoxjob/names_test.go b/internal/librivoxjob/names_test.go new file mode 100644 index 0000000..2806d07 --- /dev/null +++ b/internal/librivoxjob/names_test.go @@ -0,0 +1,212 @@ +package librivoxjob + +import ( + "net/url" + "strings" + "testing" + "unicode/utf8" + + "github.com/kodestar/audiosilo-bridge/internal/link" + "github.com/kodestar/audiosilo-bridge/internal/metaops" +) + +func TestSegment(t *testing.T) { + long := strings.Repeat("a", 300) + wide := strings.Repeat("é", 200) // 2 bytes each + for _, tc := range []struct{ in, want string }{ + {"Johnny Crow's Garden", "Johnny Crow's Garden"}, + {"../../etc/passwd", "-..-etc-passwd"}, + {"..", ""}, + {".hack", "hack"}, + {"...And Ladies of the Club", "And Ladies of the Club"}, + {" ", ""}, + {"", ""}, + {"Title: Subtitle", "Title - Subtitle"}, + {"a\\b", "a-b"}, + {"tab\there\nnewline\x00nul", "tab here newline nul"}, + {"two spaces", "two spaces"}, + {long, strings.Repeat("a", maxName)}, + } { + if got := segment(tc.in); got != tc.want { + t.Errorf("segment(%q) = %q, want %q", tc.in, got, tc.want) + } + } + got := segment(wide) + if len(got) > maxName || !utf8.ValidString(got) || len(got) != maxName { + t.Errorf("a multi-byte over-long name cut to %d bytes (valid %v)", len(got), utf8.ValidString(got)) + } + // Cut ending in a space or a dot is trimmed again. + if got := segment(strings.Repeat("a", maxName-1) + " b"); got != strings.Repeat("a", maxName-1) { + t.Errorf("a cut name kept its trailing space: %q", got) + } +} + +func TestBookNames(t *testing.T) { + book := metaops.LibrivoxBook{Title: "Metamorphosis (version 2)", + Authors: []metaops.LibrivoxPerson{{FirstName: "Franz", LastName: "Kafka"}}} + one := &metaops.LibrivoxWork{Title: "Metamorphosis", Authors: []string{"Franz Kafka"}, + Recordings: []metaops.LibrivoxRecording{{ID: "r1", LibrivoxID: 527}, {ID: "audible", LibrivoxID: 0}}} + two := &metaops.LibrivoxWork{Title: "Metamorphosis", Authors: []string{"Franz Kafka"}, + Recordings: []metaops.LibrivoxRecording{{ID: "r1", LibrivoxID: 527}, {ID: "r2", LibrivoxID: 5214}}} + for _, tc := range []struct { + name string + book metaops.LibrivoxBook + work *metaops.LibrivoxWork + wantAuthor, wantTit string + }{ + {"one LibriVox recording: the work's title", book, one, "Franz Kafka", "Metamorphosis"}, + {"two: the project's own title", book, two, "Franz Kafka", "Metamorphosis (version 2)"}, + {"no work: the project's", book, nil, "Franz Kafka", "Metamorphosis (version 2)"}, + {"empty work title: the project's", book, &metaops.LibrivoxWork{Authors: []string{"Franz Kafka"}}, "Franz Kafka", "Metamorphosis (version 2)"}, + {"empty everything", metaops.LibrivoxBook{}, nil, "", "LibriVox 192"}, + {"dot titles", metaops.LibrivoxBook{Title: "..."}, &metaops.LibrivoxWork{Title: ".."}, "", "LibriVox 192"}, + {"first credit", book, &metaops.LibrivoxWork{Title: "T", Authors: []string{"A", "B", "C"}}, "A", "T"}, + {"Various over three", book, &metaops.LibrivoxWork{Title: "T", Authors: []string{"A", "B", "C", "D"}}, "Various", "T"}, + {"Various from the feed", metaops.LibrivoxBook{Title: "Short Story Collection Vol. 001", Authors: []metaops.LibrivoxPerson{ + {LastName: "A"}, {LastName: "B"}, {LastName: "C"}, {LastName: "D"}}}, nil, "Various", "Short Story Collection Vol. 001"}, + {"author traversal", book, &metaops.LibrivoxWork{Title: "T", Authors: []string{"../x"}}, "-x", "T"}, + } { + t.Run(tc.name, func(t *testing.T) { + a, ti := bookNames(192, tc.book, tc.work) + if a != tc.wantAuthor || ti != tc.wantTit { + t.Fatalf("got %q / %q, want %q / %q", a, ti, tc.wantAuthor, tc.wantTit) + } + }) + } +} + +func TestDestination(t *testing.T) { + books := []link.LibraryBook{{Path: "L.Leslie Brooke/Other Book", Authors: []string{"L. Leslie Brooke"}}} + for _, tc := range []struct { + author, title, parent, dest string + }{ + {"L. Leslie Brooke", "Johnny Crow's Garden", "L.Leslie Brooke", "L.Leslie Brooke/Johnny Crow's Garden"}, + {"Franz Kafka", "Metamorphosis (version 2)", "Franz Kafka", "Franz Kafka/Metamorphosis (version 2)"}, + {"", "Anonymous Tales", "", "Anonymous Tales"}, + {"Various", "Short Story Collection Vol. 001", "Various", "Various/Short Story Collection Vol. 001"}, + } { + parent, dest := destination(tc.author, tc.title, books) + if parent != tc.parent || dest != tc.dest { + t.Errorf("destination(%q, %q) = %q, %q; want %q, %q", tc.author, tc.title, parent, dest, tc.parent, tc.dest) + } + } +} + +func TestSectionNames(t *testing.T) { + sections := []metaops.LibrivoxSection{ + {Title: "Introduction", ListenURL: "https://archive.org/download/x/intro_64kb.mp3"}, + {Title: "", ListenURL: "https://archive.org/download/x/one.ogg"}, + {Title: "../../escape", ListenURL: "https://archive.org/download/x/two"}, + {Title: ".hidden", ListenURL: "https://archive.org/download/x/three.MP3?x=1"}, + {Title: strings.Repeat("w", 400), ListenURL: "https://archive.org/download/x/four.exe"}, + } + got := sectionNames(sections) + want := []string{ + "01 - Introduction.mp3", + "02 - Section 2.ogg", + "03 - -..-escape.mp3", + "04 - hidden.mp3", + "05 - " + strings.Repeat("w", maxName) + ".mp3", + } + for i := range want { + if got[i] != want[i] { + t.Errorf("section %d: %q, want %q", i+1, got[i], want[i]) + } + if len(got[i]+partSuffix) > 255 { + t.Errorf("section %d's partial name is %d bytes", i+1, len(got[i]+partSuffix)) + } + } + // The width is the count's, at least two digits. + many := make([]metaops.LibrivoxSection, 120) + names := sectionNames(many) + if names[0] != "001 - Section 1.mp3" || names[119] != "120 - Section 120.mp3" { + t.Errorf("120 sections: %q .. %q", names[0], names[119]) + } + if one := sectionNames(many[:1]); one[0] != "01 - Section 1.mp3" { + t.Errorf("one section: %q", one[0]) + } +} + +func TestFallbackNameFits(t *testing.T) { + name := fallbackName(strings.Repeat("t", maxName), 99999999) + if len(name) > 255 { + t.Fatalf("the fallback name is %d bytes", len(name)) + } +} + +func TestAllowedURL(t *testing.T) { + for raw, want := range map[string]bool{ + "https://archive.org/download/x/1.mp3": true, + "https://www.archive.org/download/x/1.mp3": true, + "https://ia800300.us.archive.org/12/items/x/1.mp3": true, + "https://ARCHIVE.ORG/download/x/1.mp3": true, + "https://archive.org./download/x/1.mp3": true, + "https://archive.org:443/download/x/1.mp3": true, + "http://archive.org/download/x/1.mp3": false, + "http://www.archive.org/download/x/1.mp3": false, + "https://librivox.org/x": false, + "https://www.librivox.org/x": false, + "https://archive.org.evil.example/x": false, + "https://evilarchive.org/x": false, + "https://user:pw@archive.org/x": false, + "https://archive.org:8443/x": false, + "ftp://archive.org/x": false, + "https://207.241.224.2/x": false, + "//archive.org/x": false, + } { + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + if got := AllowedURL(u); got != want { + t.Errorf("AllowedURL(%s) = %v, want %v", raw, got, want) + } + } + if AllowedURL(nil) { + t.Error("a nil URL allowed") + } +} + +// A section title that opens with the section's own number (its +// section_number or its position) loses it: the file name carries it already. +func TestSectionNamesDropOwnNumber(t *testing.T) { + for _, tc := range []struct { + title, number string + pos int + want string + }{ + {"1 - Chapter 01 through Chapter 06", "1", 1, "Chapter 01 through Chapter 06"}, // project 5495 + {"1. Down the Rabbit-Hole", "1", 1, "Down the Rabbit-Hole"}, + {"2: The Pool of Tears", "2", 2, "The Pool of Tears"}, + {"3) A Caucus-Race", "3", 3, "A Caucus-Race"}, + {"03 - Zero padded", "3", 3, "Zero padded"}, + {"0 - Preface", "0", 1, "Preface"}, // section_number from 0 + {"1 - Preface", "0", 1, "Preface"}, // the position + {"7 - Wrong number", "3", 3, "7 - Wrong number"}, // neither: kept + {"1984", "1", 1, "1984"}, // a number title, not a prefix + {"1984 - Part One", "1", 1, "1984 - Part One"}, + {"1.5 Interlude", "1", 1, "1.5 Interlude"}, // no separator before a word + {"12 Angry Men", "12", 12, "12 Angry Men"}, // no separator + {"5", "5", 5, ""}, // only its own number + {"5.", "5", 5, ""}, // nothing after the separator + {"5", "", 5, ""}, + {"Chapter 1", "1", 1, "Chapter 1"}, + {"", "1", 1, ""}, + } { + if got := dropOwnNumber(tc.title, tc.number, tc.pos); got != tc.want { + t.Errorf("dropOwnNumber(%q, %q, %d) = %q, want %q", tc.title, tc.number, tc.pos, got, tc.want) + } + } + got := sectionNames([]metaops.LibrivoxSection{ + {SectionNumber: "1", Title: "1 - Chapter 01 through Chapter 06"}, + {SectionNumber: "2", Title: "2"}, + {SectionNumber: "3", Title: "1984"}, + {SectionNumber: "4", Title: "Introduction"}, + }) + want := []string{"01 - Chapter 01 through Chapter 06.mp3", "02 - Section 2.mp3", "03 - 1984.mp3", "04 - Introduction.mp3"} + for i := range want { + if got[i] != want[i] { + t.Errorf("section %d: %q, want %q", i+1, got[i], want[i]) + } + } +} diff --git a/internal/librivoxjob/state.go b/internal/librivoxjob/state.go new file mode 100644 index 0000000..7b13ae8 --- /dev/null +++ b/internal/librivoxjob/state.go @@ -0,0 +1,110 @@ +package librivoxjob + +import ( + "encoding/json" + "errors" + "io/fs" + "os" + "path" + "path/filepath" + "strconv" + + "github.com/kodestar/audiosilo-bridge/internal/fsutil" + "github.com/kodestar/audiosilo-bridge/internal/transfer" +) + +// state is a fetch's plan and progress, kept in the Bridge's data +// (<StateDir>/<library_id>-<librivox_id>.json, one per library and project, so +// fetches of one project into two libraries never touch each other's staging) +// from the moment it picks the folder: a resumed run (a restart, a lost claim +// handed out again) goes on with the same staging folder and destination +// instead of planning again against a library that changed meanwhile, Abandon +// finds the staging folder to remove, and once the folder is placed it is the +// placement marker (a failed rescan retries from the rescan, decision 6). +type state struct { + LibraryID int64 `json:"library_id"` + // Parent is the folder the book goes in ("" for the library's root), and + // MadeParent whether this fetch created it (removed again, when empty, with + // the staging folder). + Parent string `json:"parent"` + MadeParent bool `json:"made_parent,omitempty"` + // Dest is the book's folder, directly in Parent (as the staging folder is). + Dest string `json:"dest"` + // Files is each section's partial download: the strong ETag and the + // size its response stated, so a resume asks only for the rest. + Files map[string]partState `json:"files,omitempty"` + // Placed: the staging folder became Dest, holding Count files of Bytes. + Placed bool `json:"placed,omitempty"` + Count int `json:"count,omitempty"` + Bytes int64 `json:"bytes,omitempty"` +} + +// partState is one section's partial download. +type partState struct { + ETag string `json:"etag,omitempty"` + Total int64 `json:"total,omitempty"` +} + +// stagingName is the staging folder's name: a dot folder the server's scan +// never indexes, beside the book's destination so placing it is one rename. +func stagingName(id int) string { return ".audiosilo-librivox-" + strconv.Itoa(id) } + +// staging is the project's staging folder: <parent>/.audiosilo-librivox-<id>. +func (st *state) staging(id int) string { return path.Join(st.Parent, stagingName(id)) } + +// statePath is the state file of the project's fetch into library lib. +func (h *Handler) statePath(lib int64, id int) string { + return filepath.Join(h.o.StateDir, strconv.FormatInt(lib, 10)+"-"+strconv.Itoa(id)+".json") +} + +// loadState reads the state of the project's fetch into library lib, nil when +// there is none. One that can't be read, that names another library, or whose +// paths aren't clean library paths, goes: the run plans afresh. +func (h *Handler) loadState(lib int64, id int) *state { + p := h.statePath(lib, id) + b, err := os.ReadFile(p) //nolint:gosec // G304: the job's own state in the Bridge's data + if err != nil { + return nil + } + var st state + if json.Unmarshal(b, &st) != nil || st.LibraryID != lib || !st.valid() { + _ = os.Remove(p) + return nil + } + return &st +} + +// valid reports whether the state's paths are clean library paths: Parent, +// and the destination directly in it. +func (st *state) valid() bool { + if st.LibraryID <= 0 { + return false + } + parent := st.Parent + if parent == "" { + parent = "." + } else if clean, err := transfer.CleanRel(parent); err != nil || clean != parent { + return false + } + clean, err := transfer.CleanRel(st.Dest) + return err == nil && clean == st.Dest && path.Dir(st.Dest) == parent +} + +// saveState writes the state of the project's fetch into st's library. +func (h *Handler) saveState(id int, st *state) error { + if err := os.MkdirAll(h.o.StateDir, 0o700); err != nil { + return err + } + b, err := json.Marshal(st) + if err != nil { + return err + } + return fsutil.WriteFileAtomic(h.statePath(st.LibraryID, id), b, 0o600) +} + +// dropState removes the state of the project's fetch into library lib. +func (h *Handler) dropState(lib int64, id int) { + if err := os.Remove(h.statePath(lib, id)); err != nil && !errors.Is(err, fs.ErrNotExist) { + h.o.Log.Warn("librivox: removing a fetch's state failed", "library", lib, "librivox_id", id, "err", err) + } +} diff --git a/internal/metaops/coverage.go b/internal/metaops/coverage.go index fba6baf..24be336 100644 --- a/internal/metaops/coverage.go +++ b/internal/metaops/coverage.go @@ -577,6 +577,45 @@ func (c *Client) lookup(ctx context.Context, kind, value string) (workID string, return v.workID, v.found, true } +// workPayload is the works/{id} payload, the fields this package reads: its +// ONE mirror (workDetail's cached projection and LibrivoxWork's both decode it). +type workPayload struct { + ID string `json:"id"` + Title string `json:"title"` + Authors []struct { + Name string `json:"name"` + } `json:"authors"` + Series []SeriesRef `json:"series"` + // Characters carries the names too, not just presence: SeriesGlossary needs + // them, and decoding them here keeps one cache entry per work (a sibling of + // the book being processed is usually already warm from the coverage path). + Characters []struct { + Name string `json:"name"` + Aliases []string `json:"aliases"` + } `json:"characters"` + Recaps []json.RawMessage `json:"recaps"` + Recordings []struct { + ID string `json:"id"` + Narrators []struct { + Name string `json:"name"` + } `json:"narrators"` + RuntimeMin int `json:"runtime_min"` + ASINs []RegionASIN `json:"asin"` + ISBNs []string `json:"isbn"` + // LibrivoxID and ArchiveID: the recording's LibriVox project and its + // Internet Archive item (librivox.go). + LibrivoxID int `json:"librivox_id"` + ArchiveID string `json:"archive_id"` + } `json:"recordings"` +} + +// fetchWork reads works/{id}, uncached. found=false is a clean 404; ok=false a +// transport failure. +func (c *Client) fetchWork(ctx context.Context, workID string) (res workPayload, found, ok bool) { + found, ok = c.getJSON(ctx, "/api/v1/works/"+url.PathEscape(workID), &res) + return res, found, ok +} + // workDetail fetches a work's title + sidecar presence, cached per work. Sidecar // keys are omitempty on the wire, so a present, non-empty array means the work // carries that sidecar. found=false is a clean 404; ok=false is a transport @@ -585,30 +624,7 @@ func (c *Client) workDetail(ctx context.Context, workID string) (v workVal, foun if cached, hit := c.works.get(workID); hit { return cached, true, true } - var res struct { - ID string `json:"id"` - Title string `json:"title"` - Series []SeriesRef `json:"series"` - // Characters carries the names too, not just presence: SeriesGlossary needs - // them, and decoding them here keeps ONE mirror of the works/{id} payload - // and one cache entry per work (a sibling of the book being processed is - // usually already warm from the coverage path). - Characters []struct { - Name string `json:"name"` - Aliases []string `json:"aliases"` - } `json:"characters"` - Recaps []json.RawMessage `json:"recaps"` - Recordings []struct { - ID string `json:"id"` - Narrators []struct { - Name string `json:"name"` - } `json:"narrators"` - RuntimeMin int `json:"runtime_min"` - ASINs []RegionASIN `json:"asin"` - ISBNs []string `json:"isbn"` - } `json:"recordings"` - } - f, okc := c.getJSON(ctx, "/api/v1/works/"+url.PathEscape(workID), &res) + res, f, okc := c.fetchWork(ctx, workID) if !okc { return workVal{}, false, false } diff --git a/internal/metaops/librivox.go b/internal/metaops/librivox.go new file mode 100644 index 0000000..6eb1c59 --- /dev/null +++ b/internal/metaops/librivox.go @@ -0,0 +1,160 @@ +package metaops + +import ( + "context" + "errors" + "net/url" + "strconv" + "strings" +) + +// The LibriVox reads librivox.fetch makes (LIBRIVOX-PLAN.md L6, decision 5): +// the project's manifest from meta's LibriVox-shaped feed +// (GET /api/v1/librivox/audiobooks?id=N&extended=1&coverart=1, LibriVox's own +// shape) and the work it belongs to (GET /api/v1/works/{id}), uncached: a +// fetch reads each once per run. + +// ErrNotFound is a clean 404 from meta: no such project or work. +var ErrNotFound = errors.New("not found in the metadata database") + +// ErrUnavailable is meta unreachable or answering something unreadable (a +// transport failure, a 5xx, a body that isn't the shape). +var ErrUnavailable = errors.New("the metadata service is unavailable") + +// LibrivoxBook is one project of the LibriVox-shaped feed, the fields a fetch +// reads. LibriVox's ids are strings, and coverart_jpg is null when the project +// states none. +type LibrivoxBook struct { + ID string `json:"id"` + Title string `json:"title"` + Authors []LibrivoxPerson `json:"authors"` + URLIArchive string `json:"url_iarchive"` + CoverartJPG *string `json:"coverart_jpg"` + Sections []LibrivoxSection `json:"sections"` +} + +// LibrivoxPerson is an author as the feed names one (the stored name split at +// its last word). +type LibrivoxPerson struct { + FirstName string `json:"first_name"` + LastName string `json:"last_name"` +} + +// Name is the person's name as stored ("Franz Kafka", "Voltaire"). +func (p LibrivoxPerson) Name() string { + return strings.TrimSpace(strings.TrimSpace(p.FirstName) + " " + strings.TrimSpace(p.LastName)) +} + +// LibrivoxSection is one section (one file) of a project. +type LibrivoxSection struct { + // SectionNumber is LibriVox's own number for the section (from 0 on some + // projects), a string as LibriVox sends it. + SectionNumber string `json:"section_number"` + Title string `json:"title"` + ListenURL string `json:"listen_url"` +} + +// librivoxDetails is the prefix the feed states url_iarchive with. +const librivoxDetails = "/details/" + +// ArchiveID is the Internet Archive item of the project, from url_iarchive +// ("https://www.archive.org/details/<item>"), "" when it states none. +func (b LibrivoxBook) ArchiveID() string { + u, err := url.Parse(strings.TrimSpace(b.URLIArchive)) + if err != nil { + return "" + } + _, item, ok := strings.Cut(u.Path, librivoxDetails) + if !ok { + return "" + } + item, _, _ = strings.Cut(item, "/") + return item +} + +// LibrivoxAudiobook reads LibriVox project id from meta's feed, with its +// sections and cover. ErrDisabled: no base URL; ErrNotFound: meta has no such +// project; ErrUnavailable: meta can't be read now. +func (c *Client) LibrivoxAudiobook(ctx context.Context, id int) (LibrivoxBook, error) { + if !c.Enabled() { + return LibrivoxBook{}, ErrDisabled + } + q := url.Values{"id": {strconv.Itoa(id)}, "extended": {"1"}, "coverart": {"1"}} + var res struct { + Books []LibrivoxBook `json:"books"` + } + found, ok := c.getJSON(ctx, "/api/v1/librivox/audiobooks?"+q.Encode(), &res) + if err := readErr(ctx, found, ok); err != nil { + return LibrivoxBook{}, err + } + for _, b := range res.Books { + if b.ID == strconv.Itoa(id) { + return b, nil + } + } + return LibrivoxBook{}, ErrNotFound +} + +// LibrivoxWork is a work as a fetch reads it: its title, its authors' names +// and its recordings' LibriVox identities. +type LibrivoxWork struct { + Title string + Authors []string + Recordings []LibrivoxRecording +} + +// LibrivoxRecording is one recording of a work: its id, the LibriVox project +// it is (0: not a LibriVox recording, or a metaserve too old to say) and its +// Internet Archive item. +type LibrivoxRecording struct { + ID string + LibrivoxID int + ArchiveID string +} + +// LibrivoxCount is how many of the work's recordings are LibriVox projects. +func (w LibrivoxWork) LibrivoxCount() int { + n := 0 + for _, r := range w.Recordings { + if r.LibrivoxID > 0 { + n++ + } + } + return n +} + +// readErr is a getJSON outcome as these reads' error: the caller's cancel, +// ErrUnavailable, ErrNotFound, or nil. +func readErr(ctx context.Context, found, ok bool) error { + switch { + case ctx.Err() != nil: + return ctx.Err() + case !ok: + return ErrUnavailable + case !found: + return ErrNotFound + } + return nil +} + +// LibrivoxWork reads a work for a LibriVox fetch. ErrDisabled, ErrNotFound and +// ErrUnavailable as LibrivoxAudiobook's. +func (c *Client) LibrivoxWork(ctx context.Context, workID string) (LibrivoxWork, error) { + if !c.Enabled() { + return LibrivoxWork{}, ErrDisabled + } + res, found, ok := c.fetchWork(ctx, workID) + if err := readErr(ctx, found, ok); err != nil { + return LibrivoxWork{}, err + } + w := LibrivoxWork{Title: res.Title} + for _, a := range res.Authors { + if name := strings.TrimSpace(a.Name); name != "" { + w.Authors = append(w.Authors, name) + } + } + for _, r := range res.Recordings { + w.Recordings = append(w.Recordings, LibrivoxRecording{ID: r.ID, LibrivoxID: r.LibrivoxID, ArchiveID: r.ArchiveID}) + } + return w, nil +} diff --git a/internal/metaops/librivox_test.go b/internal/metaops/librivox_test.go new file mode 100644 index 0000000..6370686 --- /dev/null +++ b/internal/metaops/librivox_test.go @@ -0,0 +1,112 @@ +package metaops + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +// feed192 is meta's LibriVox-shaped answer for project 192 (extended=1, +// coverart=1), in LibriVox's own shape: string ids, numbers and playtimes. +const feed192 = `{"books":[{"id":"192","title":"Johnny Crow's Garden","description":"","url_text_source":"", +"language":"English","copyright_year":"1903","num_sections":"2","url_rss":"https://librivox.org/rss/192", +"url_zip_file":"https://archive.org/compress/johnny_crow_garden_librivox/x.zip","url_project":"", +"url_librivox":"https://librivox.org/johnny-crows-garden-by-l-leslie-brooke/","url_other":"", +"totaltime":"0:12:00","totaltimesecs":720, +"authors":[{"id":"404","first_name":"L. Leslie","last_name":"Brooke","dob":"1862","dod":"1940"}], +"coverart_jpg":"https://archive.org/download/johnny_crow_garden_librivox/johnnycrow_1104.jpg","coverart_pdf":null,"coverart_thumbnail":null, +"url_iarchive":"https://www.archive.org/details/johnny_crow_garden_librivox", +"sections":[{"id":"","section_number":"1","title":"Johnny Crow's Garden","listen_url":"https://www.archive.org/download/johnny_crow_garden_librivox/johnnycrow_01_brooke_64kb.mp3","language":"English","playtime":"300","file_name":null,"readers":[]}, +{"id":"","section_number":"2","title":"Johnny Crow's Party","listen_url":"https://www.archive.org/download/johnny_crow_garden_librivox/johnnycrow_02_brooke_64kb.mp3","language":"English","playtime":"420","file_name":null,"readers":[]}], +"genres":[],"translators":[]}]}` + +const work192 = `{"id":"johnny-crows-garden-brooke","title":"Johnny Crow's Garden","authors":[{"id":"l-leslie-brooke","name":"L. Leslie Brooke"}], +"language":"en","series":[],"recordings":[ +{"id":"johnny-crows-garden-brooke-librivox-192","narrators":[],"asin":[],"isbn":[],"chapter_count":2,"librivox_id":192,"archive_id":"johnny_crow_garden_librivox"}, +{"id":"johnny-crows-garden-brooke-audible","narrators":[],"asin":[],"isbn":[],"chapter_count":0}]}` + +func TestLibrivoxReads(t *testing.T) { + var queries []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + queries = append(queries, r.URL.String()) + switch { + case r.URL.Path == "/api/v1/librivox/audiobooks" && r.URL.Query().Get("id") == "192": + _, _ = w.Write([]byte(feed192)) + case r.URL.Path == "/api/v1/librivox/audiobooks" && r.URL.Query().Get("id") == "500": + http.Error(w, "boom", http.StatusInternalServerError) + case r.URL.Path == "/api/v1/librivox/audiobooks": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":"Audiobooks could not be found"}`)) + case r.URL.Path == "/api/v1/works/johnny-crows-garden-brooke": + _, _ = w.Write([]byte(work192)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + c := NewClient(srv.URL + "/") + ctx := context.Background() + + b, err := c.LibrivoxAudiobook(ctx, 192) + if err != nil { + t.Fatal(err) + } + if b.ID != "192" || b.Title != "Johnny Crow's Garden" || len(b.Sections) != 2 || b.CoverartJPG == nil || + b.Sections[1].ListenURL != "https://www.archive.org/download/johnny_crow_garden_librivox/johnnycrow_02_brooke_64kb.mp3" || + b.Sections[1].SectionNumber != "2" { + t.Fatalf("%+v", b) + } + if b.Authors[0].Name() != "L. Leslie Brooke" || b.ArchiveID() != "johnny_crow_garden_librivox" { + t.Fatalf("author %q item %q", b.Authors[0].Name(), b.ArchiveID()) + } + if queries[0] != "/api/v1/librivox/audiobooks?coverart=1&extended=1&id=192" { + t.Fatalf("asked %s", queries[0]) + } + if _, err := c.LibrivoxAudiobook(ctx, 7); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing project: %v", err) + } + if _, err := c.LibrivoxAudiobook(ctx, 500); !errors.Is(err, ErrUnavailable) { + t.Fatalf("a 500: %v", err) + } + + w, err := c.LibrivoxWork(ctx, "johnny-crows-garden-brooke") + if err != nil { + t.Fatal(err) + } + if w.Title != "Johnny Crow's Garden" || len(w.Authors) != 1 || w.Authors[0] != "L. Leslie Brooke" || w.LibrivoxCount() != 1 || + w.Recordings[0].ArchiveID != "johnny_crow_garden_librivox" { + t.Fatalf("%+v", w) + } + if _, err := c.LibrivoxWork(ctx, "nope"); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing work: %v", err) + } + + off := NewClient("") + if _, err := off.LibrivoxAudiobook(ctx, 192); !errors.Is(err, ErrDisabled) { + t.Fatalf("disabled: %v", err) + } + if _, err := off.LibrivoxWork(ctx, "x"); !errors.Is(err, ErrDisabled) { + t.Fatalf("disabled: %v", err) + } + down := NewClient("http://127.0.0.1:1") + if _, err := down.LibrivoxAudiobook(ctx, 192); !errors.Is(err, ErrUnavailable) { + t.Fatalf("unreachable: %v", err) + } +} + +func TestLibrivoxArchiveID(t *testing.T) { + for raw, want := range map[string]string{ + "https://www.archive.org/details/x_librivox": "x_librivox", + "https://archive.org/details/x.y-z/files": "x.y-z", + "https://librivox.org/not-an-item/": "", + "": "", + "https://www.archive.org/details/": "", + "https://www.archive.org/details/a?b=c#d": "a", + } { + if got := (LibrivoxBook{URLIArchive: raw}).ArchiveID(); got != want { + t.Errorf("ArchiveID(%q) = %q, want %q", raw, got, want) + } + } +} diff --git a/internal/netguard/allow_test.go b/internal/netguard/allow_test.go new file mode 100644 index 0000000..c267678 --- /dev/null +++ b/internal/netguard/allow_test.go @@ -0,0 +1,91 @@ +package netguard + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "strings" + "sync/atomic" + "testing" +) + +// TestRequestAllowFirstURLAndRedirects: a request's Allow is asked about the +// first URL and every redirect's; a refusal fails it as blocked_host before +// the refused address is reached, for Stream and Fetch alike. +func TestRequestAllowFirstURLAndRedirects(t *testing.T) { + var base string + var reachedOther atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Host != "" && strings.HasPrefix(r.Host, "other.example"): + reachedOther.Add(1) + fmt.Fprint(w, "other") + case r.URL.Path == "/to-allowed": + http.Redirect(w, r, strings.Replace(base, "127.0.0.1", "allowed.example", 1)+"/body", http.StatusFound) + case r.URL.Path == "/to-other": + http.Redirect(w, r, strings.Replace(base, "127.0.0.1", "other.example", 1)+"/body", http.StatusFound) + default: + fmt.Fprint(w, "body") + } + })) + defer server.Close() + base = server.URL + c := localClient(t) + c.resolver = resolverFunc(func(_ context.Context, _, host string) ([]netip.Addr, error) { + if strings.HasSuffix(host, ".example") { + host = "127.0.0.1" + } + ip, err := netip.ParseAddr(host) + return []netip.Addr{ip}, err + }) + // Admits 127.0.0.1 and allowed.example only. + allow := func(u *url.URL) bool { return u.Hostname() == "127.0.0.1" || u.Hostname() == "allowed.example" } + other := strings.Replace(base, "127.0.0.1", "other.example", 1) + + t.Run("allowed", func(t *testing.T) { + for _, p := range []string{"/body", "/to-allowed"} { + r, err := c.Stream(context.Background(), StreamRequest{URL: base + p, Allow: allow}) + if err != nil { + t.Fatal(p, err) + } + b, _ := io.ReadAll(r.Body) + r.Body.Close() + if string(b) != "body" { + t.Fatalf("%s: %q", p, b) + } + if !strings.HasPrefix(r.ContentType, "text/plain") { + t.Fatalf("%s: content type %q", p, r.ContentType) + } + f, err := c.Fetch(context.Background(), Request{URL: base + p, MaxBytes: 100, Allow: allow}) + if err != nil || string(f.Body) != "body" { + t.Fatal(p, err) + } + } + }) + t.Run("denied", func(t *testing.T) { + for _, raw := range []string{other + "/body", base + "/to-other"} { + _, err := c.Stream(context.Background(), StreamRequest{URL: raw, Allow: allow}) + errorCode(t, err, "blocked_host") + _, err = c.Fetch(context.Background(), Request{URL: raw, MaxBytes: 100, Allow: allow}) + errorCode(t, err, "blocked_host") + } + if n := reachedOther.Load(); n != 0 { + t.Fatalf("a refused host was reached %d times", n) + } + }) + t.Run("no allow keeps the old rule", func(t *testing.T) { + r, err := c.Stream(context.Background(), StreamRequest{URL: base + "/to-other"}) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(r.Body) + r.Body.Close() + if string(b) != "other" { + t.Fatalf("%q", b) + } + }) +} diff --git a/internal/netguard/fetch.go b/internal/netguard/fetch.go index 54d37d6..50e9369 100644 --- a/internal/netguard/fetch.go +++ b/internal/netguard/fetch.go @@ -31,6 +31,9 @@ type Request struct { ETag string LastModified string MaxBytes int64 + // Allow is StreamRequest.Allow: asked about the first URL and every + // redirect's, a refusal fails the request as blocked_host. + Allow func(*url.URL) bool } type Response struct { Status int @@ -62,6 +65,9 @@ func (c *Client) Fetch(ctx context.Context, in Request) (*Response, error) { if in.MaxBytes <= 0 { return nil, &Error{Code: "too_large", Message: "The response size limit is invalid."} } + if !allowedBy(in.Allow, in.URL) { + return nil, blockedHost() + } ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() for attempt := 0; ; attempt++ { @@ -93,6 +99,9 @@ func (c *Client) once(ctx context.Context, in Request) (*Response, error) { if !validURL(req.URL.String()) { return &Error{Code: "invalid_feed_url", Message: "The feed redirected to an invalid URL."} } + if !allows(in.Allow, req.URL) { + return blockedHost() + } stripRedirectCredentials(req, via) redirected = true if req.Response.StatusCode != 301 && req.Response.StatusCode != 308 { diff --git a/internal/netguard/guard.go b/internal/netguard/guard.go index cdd8d54..84020fd 100644 --- a/internal/netguard/guard.go +++ b/internal/netguard/guard.go @@ -130,6 +130,27 @@ func (c *Client) dial(ctx context.Context, network, address string) (net.Conn, e return nil, err } +// allowedBy reports whether allow (nil: every URL) admits raw, a request's +// first URL. +func allowedBy(allow func(*url.URL) bool, raw string) bool { + if allow == nil { + return true + } + u, err := url.Parse(raw) + return err == nil && allow(u) +} + +// allows reports whether allow (nil: every URL) admits u, a redirect's URL. +func allows(allow func(*url.URL) bool, u *url.URL) bool { + return allow == nil || allow(u) +} + +// blockedHost is a request's failure when its Allow refused a URL, the first +// or a redirect's. +func blockedHost() *Error { + return &Error{Code: "blocked_host", Message: "The address is not on the hosts this request may reach."} +} + func validURL(raw string) bool { u, err := url.Parse(raw) return err == nil && u.Hostname() != "" && (u.Scheme == "http" || u.Scheme == "https") diff --git a/internal/netguard/stream.go b/internal/netguard/stream.go index e17bffe..5c94f79 100644 --- a/internal/netguard/stream.go +++ b/internal/netguard/stream.go @@ -4,6 +4,7 @@ import ( "context" "io" "net/http" + "net/url" "strconv" "strings" "sync" @@ -14,13 +15,21 @@ type StreamRequest struct { URL string Offset int64 ETag string + // Allow, when set, is asked about the first URL and every redirect's: a + // URL it refuses fails the request as blocked_host before it is sent + // (librivox.fetch: archive.org over https only). The address checks apply + // either way. + Allow func(*url.URL) bool } type StreamResponse struct { - Status int - Body io.ReadCloser - ContentLength int64 - ContentRange string - Encoding string + Status int + Body io.ReadCloser + ContentLength int64 + ContentRange string + Encoding string + // ContentType is the response's Content-Type as sent (librivox.fetch + // refuses a page where a file was asked for). + ContentType string ETag string RetryAfterSeconds int } @@ -31,6 +40,9 @@ func (c *Client) Stream(ctx context.Context, in StreamRequest) (*StreamResponse, if !validURL(in.URL) { return nil, &Error{Code: "invalid_feed_url", Message: "The episode address is invalid."} } + if !allowedBy(in.Allow, in.URL) { + return nil, blockedHost() + } ctx, cancel := context.WithCancel(ctx) hc := &http.Client{Transport: c.transport, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) > 10 { @@ -39,6 +51,9 @@ func (c *Client) Stream(ctx context.Context, in StreamRequest) (*StreamResponse, if !validURL(req.URL.String()) { return &Error{Code: "invalid_feed_url", Message: "The episode redirected to an invalid address."} } + if !allows(in.Allow, req.URL) { + return blockedHost() + } stripRedirectCredentials(req, via) return nil }} @@ -60,7 +75,7 @@ func (c *Client) Stream(ctx context.Context, in StreamRequest) (*StreamResponse, } timer := time.AfterFunc(c.stall, cancel) body := &streamBody{progressReader: progressReader{r: resp.Body, timer: timer, stall: c.stall}, body: resp.Body, cancel: cancel} - return &StreamResponse{Status: resp.StatusCode, Body: body, ContentLength: resp.ContentLength, ContentRange: resp.Header.Get("Content-Range"), Encoding: strings.ToLower(strings.TrimSpace(resp.Header.Get("Content-Encoding"))), ETag: resp.Header.Get("ETag"), RetryAfterSeconds: retryAfter(resp.Header.Get("Retry-After"), time.Now())}, nil + return &StreamResponse{Status: resp.StatusCode, Body: body, ContentLength: resp.ContentLength, ContentRange: resp.Header.Get("Content-Range"), Encoding: strings.ToLower(strings.TrimSpace(resp.Header.Get("Content-Encoding"))), ContentType: resp.Header.Get("Content-Type"), ETag: resp.Header.Get("ETag"), RetryAfterSeconds: retryAfter(resp.Header.Get("Retry-After"), time.Now())}, nil } // streamBody resets the stall timer on progress (progressReader) and ends the diff --git a/internal/placement/placement.go b/internal/placement/placement.go index 0247e3f..1f93457 100644 --- a/internal/placement/placement.go +++ b/internal/placement/placement.go @@ -97,7 +97,7 @@ func Suggest(in Input) Suggestion { func standalone(in Input) Suggestion { return Suggestion{ - RelPath: authorSegment(in) + "/" + sanitize(in.Title), + RelPath: authorSegment(in) + "/" + Sanitize(in.Title), Filename: in.Filename, Kind: Standalone, Note: "no series - placed directly under the author", @@ -111,7 +111,7 @@ func authorSegment(in Input) string { if in.AuthorFolder != "" { return in.AuthorFolder } - return sanitize(in.Author) + return Sanitize(in.Author) } func seriesWithSiblings(in Input) Suggestion { @@ -155,7 +155,7 @@ func seriesNew(in Input) Suggestion { } prefix := guessShortcode(in.SeriesName) leaf := buildLeaf(prefix, 2, in) - rel := authorSegment(in) + "/" + sanitize(in.SeriesName) + "/" + leaf + rel := authorSegment(in) + "/" + Sanitize(in.SeriesName) + "/" + leaf return Suggestion{ RelPath: rel, Filename: in.Filename, @@ -168,10 +168,10 @@ func seriesNew(in Input) Suggestion { // sequence it falls back to just the title (still inside its own folder). func buildLeaf(prefix string, width int, in Input) string { if !in.HasSequence { - return sanitize(in.Title) + return Sanitize(in.Title) } num := formatSequence(in.Sequence, width) - return sanitize(prefix + num + " - " + in.Title) + return Sanitize(prefix + num + " - " + in.Title) } // bareTitle reports whether a cleaned title has nothing of its own to name a folder @@ -416,11 +416,12 @@ var stopwords = map[string]bool{ "to": true, "in": true, } -// sanitize makes a path segment safe to create on disk without mangling the +// Sanitize makes a path segment safe to create on disk without mangling the // punctuation that real library folders use (apostrophes, commas, periods). // A leading dot goes too ("...And Ladies of the Club", ".hack"): a hidden -// name is never indexed by the server's scan. -func sanitize(s string) string { +// name is never indexed by the server's scan. librivox.fetch names its +// section files with it too, so a book's folder and its files follow one rule. +func Sanitize(s string) string { s = strings.TrimSpace(s) replacer := strings.NewReplacer("/", "-", "\\", "-", ":", " -", "\x00", "") s = replacer.Replace(s) diff --git a/internal/placement/template.go b/internal/placement/template.go index 3576884..e174122 100644 --- a/internal/placement/template.go +++ b/internal/placement/template.go @@ -224,7 +224,7 @@ func (t *Template) Render(v Vars) (string, error) { parts := strings.Split(sb.String(), "/") out := make([]string, 0, len(parts)) for _, p := range parts { - if s := sanitize(p); s != "" { + if s := Sanitize(p); s != "" { out = append(out, s) } } diff --git a/web/src/lib/generate.test.ts b/web/src/lib/generate.test.ts index c23bf06..4b6d91c 100644 --- a/web/src/lib/generate.test.ts +++ b/web/src/lib/generate.test.ts @@ -136,6 +136,7 @@ describe('descriptions', () => { expect(kindLabel('podcast.fetch')).toBe('Fetching an episode'); expect(kindLabel('podcast.retain')).toBe('Checking retained episodes'); expect(kindLabel('audible.poll')).toBe('Checking an Audible library'); + expect(kindLabel('librivox.fetch')).toBe('Adding from LibriVox'); expect(kindLabel('sidecars.contribute')).toBe('Community share'); expect(kindLabel('meta.correct')).toBe('Corrections'); expect(kindLabel('bridge.verify')).toBe('Verify'); diff --git a/web/src/lib/generate.ts b/web/src/lib/generate.ts index d47e6b5..96691b0 100644 --- a/web/src/lib/generate.ts +++ b/web/src/lib/generate.ts @@ -57,6 +57,7 @@ const KIND_LABELS: Record<string, string> = { 'podcast.fetch': 'Fetching an episode', 'podcast.retain': 'Checking retained episodes', 'audible.poll': 'Checking an Audible library', + 'librivox.fetch': 'Adding from LibriVox', }; // kindLabel words a job kind ("" for one with no label: the kind shows as it is).