From 45cbc093edd67a225be129aa3974b456861aa68e Mon Sep 17 00:00:00 2001 From: KodeStar Date: Fri, 13 Nov 2026 16:11:49 +0000 Subject: [PATCH 01/22] netguard: check a per-request host allowlist on the first URL and every redirect Request and StreamRequest take an optional Allow, asked about the first URL and each redirect hop in CheckRedirect; a refusal fails as blocked_host before the refused host is dialled. Nil keeps today's rule, so podcasts are unchanged. librivox.fetch uses it to stay on archive.org. Part of LibriVox L6-1 (Add from LibriVox). --- internal/netguard/allow_test.go | 88 +++++++++++++++++++++++++++++++++ internal/netguard/fetch.go | 9 ++++ internal/netguard/guard.go | 15 ++++++ internal/netguard/stream.go | 12 +++++ 4 files changed, 124 insertions(+) create mode 100644 internal/netguard/allow_test.go diff --git a/internal/netguard/allow_test.go b/internal/netguard/allow_test.go new file mode 100644 index 0000000..5b82099 --- /dev/null +++ b/internal/netguard/allow_test.go @@ -0,0 +1,88 @@ +package netguard + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "strings" + "sync/atomic" + "testing" +) + +// TestRequestAllowFirstURLAndRedirects: a request's Allow is asked about the +// first URL and every redirect's; a refusal fails it as blocked_host before +// the refused address is reached, for Stream and Fetch alike. +func TestRequestAllowFirstURLAndRedirects(t *testing.T) { + var base string + var reachedOther atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Host != "" && strings.HasPrefix(r.Host, "other.example"): + reachedOther.Add(1) + fmt.Fprint(w, "other") + case r.URL.Path == "/to-allowed": + http.Redirect(w, r, strings.Replace(base, "127.0.0.1", "allowed.example", 1)+"/body", http.StatusFound) + case r.URL.Path == "/to-other": + http.Redirect(w, r, strings.Replace(base, "127.0.0.1", "other.example", 1)+"/body", http.StatusFound) + default: + fmt.Fprint(w, "body") + } + })) + defer server.Close() + base = server.URL + c := localClient(t) + c.resolver = resolverFunc(func(_ context.Context, _, host string) ([]netip.Addr, error) { + if strings.HasSuffix(host, ".example") { + host = "127.0.0.1" + } + ip, err := netip.ParseAddr(host) + return []netip.Addr{ip}, err + }) + // Admits 127.0.0.1 and allowed.example only. + allow := func(u *url.URL) bool { return u.Hostname() == "127.0.0.1" || u.Hostname() == "allowed.example" } + other := strings.Replace(base, "127.0.0.1", "other.example", 1) + + t.Run("allowed", func(t *testing.T) { + for _, p := range []string{"/body", "/to-allowed"} { + r, err := c.Stream(context.Background(), StreamRequest{URL: base + p, Allow: allow}) + if err != nil { + t.Fatal(p, err) + } + b, _ := io.ReadAll(r.Body) + r.Body.Close() + if string(b) != "body" { + t.Fatalf("%s: %q", p, b) + } + f, err := c.Fetch(context.Background(), Request{URL: base + p, MaxBytes: 100, Allow: allow}) + if err != nil || string(f.Body) != "body" { + t.Fatal(p, err) + } + } + }) + t.Run("denied", func(t *testing.T) { + for _, raw := range []string{other + "/body", base + "/to-other"} { + _, err := c.Stream(context.Background(), StreamRequest{URL: raw, Allow: allow}) + errorCode(t, err, "blocked_host") + _, err = c.Fetch(context.Background(), Request{URL: raw, MaxBytes: 100, Allow: allow}) + errorCode(t, err, "blocked_host") + } + if n := reachedOther.Load(); n != 0 { + t.Fatalf("a refused host was reached %d times", n) + } + }) + t.Run("no allow keeps the old rule", func(t *testing.T) { + r, err := c.Stream(context.Background(), StreamRequest{URL: base + "/to-other"}) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(r.Body) + r.Body.Close() + if string(b) != "other" { + t.Fatalf("%q", b) + } + }) +} diff --git a/internal/netguard/fetch.go b/internal/netguard/fetch.go index 54d37d6..e58a6ea 100644 --- a/internal/netguard/fetch.go +++ b/internal/netguard/fetch.go @@ -31,6 +31,9 @@ type Request struct { ETag string LastModified string MaxBytes int64 + // Allow is StreamRequest.Allow: asked about the first URL and every + // redirect's, a refusal fails the request as blocked_host. + Allow func(*url.URL) bool } type Response struct { Status int @@ -62,6 +65,9 @@ func (c *Client) Fetch(ctx context.Context, in Request) (*Response, error) { if in.MaxBytes <= 0 { return nil, &Error{Code: "too_large", Message: "The response size limit is invalid."} } + if !allowedBy(in.Allow, in.URL) { + return nil, blockedHost() + } ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() for attempt := 0; ; attempt++ { @@ -93,6 +99,9 @@ func (c *Client) once(ctx context.Context, in Request) (*Response, error) { if !validURL(req.URL.String()) { return &Error{Code: "invalid_feed_url", Message: "The feed redirected to an invalid URL."} } + if in.Allow != nil && !in.Allow(req.URL) { + return blockedHost() + } stripRedirectCredentials(req, via) redirected = true if req.Response.StatusCode != 301 && req.Response.StatusCode != 308 { diff --git a/internal/netguard/guard.go b/internal/netguard/guard.go index cdd8d54..77e0845 100644 --- a/internal/netguard/guard.go +++ b/internal/netguard/guard.go @@ -130,6 +130,21 @@ func (c *Client) dial(ctx context.Context, network, address string) (net.Conn, e return nil, err } +// allowedBy reports whether allow (nil: every URL) admits raw. +func allowedBy(allow func(*url.URL) bool, raw string) bool { + if allow == nil { + return true + } + u, err := url.Parse(raw) + return err == nil && allow(u) +} + +// blockedHost is a request's failure when its Allow refused a URL, the first +// or a redirect's. +func blockedHost() *Error { + return &Error{Code: "blocked_host", Message: "The address is not on the hosts this request may reach."} +} + func validURL(raw string) bool { u, err := url.Parse(raw) return err == nil && u.Hostname() != "" && (u.Scheme == "http" || u.Scheme == "https") diff --git a/internal/netguard/stream.go b/internal/netguard/stream.go index e17bffe..16e362d 100644 --- a/internal/netguard/stream.go +++ b/internal/netguard/stream.go @@ -4,6 +4,7 @@ import ( "context" "io" "net/http" + "net/url" "strconv" "strings" "sync" @@ -14,6 +15,11 @@ type StreamRequest struct { URL string Offset int64 ETag string + // Allow, when set, is asked about the first URL and every redirect's: a + // URL it refuses fails the request as blocked_host before it is sent + // (librivox.fetch: archive.org over https only). The address checks apply + // either way. + Allow func(*url.URL) bool } type StreamResponse struct { Status int @@ -31,6 +37,9 @@ func (c *Client) Stream(ctx context.Context, in StreamRequest) (*StreamResponse, if !validURL(in.URL) { return nil, &Error{Code: "invalid_feed_url", Message: "The episode address is invalid."} } + if !allowedBy(in.Allow, in.URL) { + return nil, blockedHost() + } ctx, cancel := context.WithCancel(ctx) hc := &http.Client{Transport: c.transport, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) > 10 { @@ -39,6 +48,9 @@ func (c *Client) Stream(ctx context.Context, in StreamRequest) (*StreamResponse, if !validURL(req.URL.String()) { return &Error{Code: "invalid_feed_url", Message: "The episode redirected to an invalid address."} } + if in.Allow != nil && !in.Allow(req.URL) { + return blockedHost() + } stripRedirectCredentials(req, via) return nil }} From bb0f4416cbcc802fb0baa0151c6a6eafd64a8056 Mon Sep 17 00:00:00 2001 From: KodeStar Date: Fri, 13 Nov 2026 16:11:49 +0000 Subject: [PATCH 02/22] Share the library listing and the segment sanitizer with other import jobs importjob.AllBooks now reads a library's paged book listing for audiblejob and the coming librivoxjob, and placement exports Sanitize so a LibriVox fetch names its section files by the same rule as its book folder. No behaviour change. Part of LibriVox L6-1 (Add from LibriVox). --- internal/audiblejob/audiblejob.go | 27 +++--------------------- internal/importjob/books.go | 35 +++++++++++++++++++++++++++++++ internal/placement/placement.go | 17 ++++++++------- internal/placement/template.go | 2 +- 4 files changed, 48 insertions(+), 33 deletions(-) create mode 100644 internal/importjob/books.go diff --git a/internal/audiblejob/audiblejob.go b/internal/audiblejob/audiblejob.go index f199678..93d0892 100644 --- a/internal/audiblejob/audiblejob.go +++ b/internal/audiblejob/audiblejob.go @@ -39,9 +39,6 @@ import ( // Kind is the job kind. const Kind = "audible.poll" -// maxBookPages bounds the library listing (500 books a page: a million books). -const maxBookPages = 2000 - // maxRuntime is the longest runtime the server takes (seconds). const maxRuntime = 1_000_000 @@ -457,30 +454,12 @@ func (h *Handler) refused(ctx context.Context, id string) *Failure { return fail(CodeSignIn, "Audible refused the account's sign-in: the next check tries again, or sign in again on the Bridge's Accounts tab.") } -// bookLister reads a library's books page by page (Server and ImportServer). -type bookLister interface { - LibraryBooks(ctx context.Context, libraryID int64, after string, limit int) (*link.LibraryBooks, error) -} - -// allBooks reads every page of the library's books. -func allBooks(ctx context.Context, srv bookLister, libraryID int64) ([]link.LibraryBook, error) { +// allBooks reads every page of the account's library's books. +func allBooks(ctx context.Context, srv importjob.BookLister, libraryID int64) ([]link.LibraryBook, error) { if libraryID <= 0 { return nil, errors.New("the account has no library: map it again on the Bridge's Accounts tab") } - var books []link.LibraryBook - after := "" - for range maxBookPages { - page, err := srv.LibraryBooks(ctx, libraryID, after, link.MaxLibraryBooks) - if err != nil { - return nil, err - } - books = append(books, page.Books...) - if page.Next == "" || page.Next == after { - return books, nil - } - after = page.Next - } - return nil, fmt.Errorf("the library has more than %d pages of books", maxBookPages) + return importjob.AllBooks(ctx, srv, libraryID) } // title is a record as the poll reports it. diff --git a/internal/importjob/books.go b/internal/importjob/books.go new file mode 100644 index 0000000..6b9b7b9 --- /dev/null +++ b/internal/importjob/books.go @@ -0,0 +1,35 @@ +package importjob + +import ( + "context" + "fmt" + + "github.com/kodestar/audiosilo-bridge/internal/link" +) + +// maxBookPages bounds the library listing (500 books a page: a million books). +const maxBookPages = 2000 + +// BookLister reads a library's books page by page (*link.Client). +type BookLister interface { + LibraryBooks(ctx context.Context, libraryID int64, after string, limit int) (*link.LibraryBooks, error) +} + +// AllBooks reads every page of the library's books (the server's paged +// listing, read whole), as Plan and the matcher take them. +func AllBooks(ctx context.Context, srv BookLister, libraryID int64) ([]link.LibraryBook, error) { + var books []link.LibraryBook + after := "" + for range maxBookPages { + page, err := srv.LibraryBooks(ctx, libraryID, after, link.MaxLibraryBooks) + if err != nil { + return nil, err + } + books = append(books, page.Books...) + if page.Next == "" || page.Next == after { + return books, nil + } + after = page.Next + } + return nil, fmt.Errorf("the library has more than %d pages of books", maxBookPages) +} diff --git a/internal/placement/placement.go b/internal/placement/placement.go index 0247e3f..1f93457 100644 --- a/internal/placement/placement.go +++ b/internal/placement/placement.go @@ -97,7 +97,7 @@ func Suggest(in Input) Suggestion { func standalone(in Input) Suggestion { return Suggestion{ - RelPath: authorSegment(in) + "/" + sanitize(in.Title), + RelPath: authorSegment(in) + "/" + Sanitize(in.Title), Filename: in.Filename, Kind: Standalone, Note: "no series - placed directly under the author", @@ -111,7 +111,7 @@ func authorSegment(in Input) string { if in.AuthorFolder != "" { return in.AuthorFolder } - return sanitize(in.Author) + return Sanitize(in.Author) } func seriesWithSiblings(in Input) Suggestion { @@ -155,7 +155,7 @@ func seriesNew(in Input) Suggestion { } prefix := guessShortcode(in.SeriesName) leaf := buildLeaf(prefix, 2, in) - rel := authorSegment(in) + "/" + sanitize(in.SeriesName) + "/" + leaf + rel := authorSegment(in) + "/" + Sanitize(in.SeriesName) + "/" + leaf return Suggestion{ RelPath: rel, Filename: in.Filename, @@ -168,10 +168,10 @@ func seriesNew(in Input) Suggestion { // sequence it falls back to just the title (still inside its own folder). func buildLeaf(prefix string, width int, in Input) string { if !in.HasSequence { - return sanitize(in.Title) + return Sanitize(in.Title) } num := formatSequence(in.Sequence, width) - return sanitize(prefix + num + " - " + in.Title) + return Sanitize(prefix + num + " - " + in.Title) } // bareTitle reports whether a cleaned title has nothing of its own to name a folder @@ -416,11 +416,12 @@ var stopwords = map[string]bool{ "to": true, "in": true, } -// sanitize makes a path segment safe to create on disk without mangling the +// Sanitize makes a path segment safe to create on disk without mangling the // punctuation that real library folders use (apostrophes, commas, periods). // A leading dot goes too ("...And Ladies of the Club", ".hack"): a hidden -// name is never indexed by the server's scan. -func sanitize(s string) string { +// name is never indexed by the server's scan. librivox.fetch names its +// section files with it too, so a book's folder and its files follow one rule. +func Sanitize(s string) string { s = strings.TrimSpace(s) replacer := strings.NewReplacer("/", "-", "\\", "-", ":", " -", "\x00", "") s = replacer.Replace(s) diff --git a/internal/placement/template.go b/internal/placement/template.go index 3576884..e174122 100644 --- a/internal/placement/template.go +++ b/internal/placement/template.go @@ -224,7 +224,7 @@ func (t *Template) Render(v Vars) (string, error) { parts := strings.Split(sb.String(), "/") out := make([]string, 0, len(parts)) for _, p := range parts { - if s := sanitize(p); s != "" { + if s := Sanitize(p); s != "" { out = append(out, s) } } From 65111ae26baf9aed0cc02babd190fa7cc1571a6d Mon Sep 17 00:00:00 2001 From: KodeStar Date: Fri, 13 Nov 2026 16:11:49 +0000 Subject: [PATCH 03/22] metaops: read a LibriVox project's manifest and its work LibrivoxAudiobook reads one project from meta's LibriVox-shaped feed (extended=1&coverart=1: sections, cover, url_iarchive) and LibrivoxWork the work's title, authors and recordings' librivox_id and archive_id. Both are uncached and tell disabled, not found and unavailable apart, which librivox.fetch maps to metadata_off, not_found and a retryable error. Part of LibriVox L6-1 (Add from LibriVox). --- internal/metaops/librivox.go | 168 ++++++++++++++++++++++++++++++ internal/metaops/librivox_test.go | 112 ++++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 internal/metaops/librivox.go create mode 100644 internal/metaops/librivox_test.go diff --git a/internal/metaops/librivox.go b/internal/metaops/librivox.go new file mode 100644 index 0000000..7aedff4 --- /dev/null +++ b/internal/metaops/librivox.go @@ -0,0 +1,168 @@ +package metaops + +import ( + "context" + "errors" + "net/url" + "strconv" + "strings" +) + +// The LibriVox reads librivox.fetch makes (LIBRIVOX-PLAN.md L6, decision 5): +// the project's manifest from meta's LibriVox-shaped feed +// (GET /api/v1/librivox/audiobooks?id=N&extended=1&coverart=1, LibriVox's own +// shape) and the work it belongs to (GET /api/v1/works/{id}), uncached: a +// fetch reads each once per run. + +// ErrNotFound is a clean 404 from meta: no such project or work. +var ErrNotFound = errors.New("not found in the metadata database") + +// ErrUnavailable is meta unreachable or answering something unreadable (a +// transport failure, a 5xx, a body that isn't the shape). +var ErrUnavailable = errors.New("the metadata service is unavailable") + +// LibrivoxBook is one project of the LibriVox-shaped feed, the fields a fetch +// reads. LibriVox's ids, section numbers and playtimes are strings, and +// coverart_jpg is null when the project states none. +type LibrivoxBook struct { + ID string `json:"id"` + Title string `json:"title"` + Authors []LibrivoxPerson `json:"authors"` + URLIArchive string `json:"url_iarchive"` + CoverartJPG *string `json:"coverart_jpg"` + Sections []LibrivoxSection `json:"sections"` +} + +// LibrivoxPerson is an author as the feed names one (the stored name split at +// its last word). +type LibrivoxPerson struct { + FirstName string `json:"first_name"` + LastName string `json:"last_name"` +} + +// Name is the person's name as stored ("Franz Kafka", "Voltaire"). +func (p LibrivoxPerson) Name() string { + return strings.TrimSpace(strings.TrimSpace(p.FirstName) + " " + strings.TrimSpace(p.LastName)) +} + +// LibrivoxSection is one section (one file) of a project. +type LibrivoxSection struct { + SectionNumber string `json:"section_number"` + Title string `json:"title"` + ListenURL string `json:"listen_url"` + Playtime string `json:"playtime"` +} + +// librivoxDetails is the prefix the feed states url_iarchive with. +const librivoxDetails = "/details/" + +// ArchiveID is the Internet Archive item of the project, from url_iarchive +// ("https://www.archive.org/details/"), "" when it states none. +func (b LibrivoxBook) ArchiveID() string { + u, err := url.Parse(strings.TrimSpace(b.URLIArchive)) + if err != nil { + return "" + } + _, item, ok := strings.Cut(u.Path, librivoxDetails) + if !ok { + return "" + } + item, _, _ = strings.Cut(item, "/") + return item +} + +// LibrivoxAudiobook reads LibriVox project id from meta's feed, with its +// sections and cover. ErrDisabled: no base URL; ErrNotFound: meta has no such +// project; ErrUnavailable: meta can't be read now. +func (c *Client) LibrivoxAudiobook(ctx context.Context, id int) (LibrivoxBook, error) { + if !c.Enabled() { + return LibrivoxBook{}, ErrDisabled + } + q := url.Values{"id": {strconv.Itoa(id)}, "extended": {"1"}, "coverart": {"1"}} + var res struct { + Books []LibrivoxBook `json:"books"` + } + found, ok := c.getJSON(ctx, "/api/v1/librivox/audiobooks?"+q.Encode(), &res) + switch { + case ctx.Err() != nil: + return LibrivoxBook{}, ctx.Err() + case !ok: + return LibrivoxBook{}, ErrUnavailable + case !found: + return LibrivoxBook{}, ErrNotFound + } + for _, b := range res.Books { + if b.ID == strconv.Itoa(id) { + return b, nil + } + } + return LibrivoxBook{}, ErrNotFound +} + +// LibrivoxWork is a work as a fetch reads it: its title, its authors' names +// and its recordings' LibriVox identities. +type LibrivoxWork struct { + ID string + Title string + Authors []string + Recordings []LibrivoxRecording +} + +// LibrivoxRecording is one recording of a work: its id, the LibriVox project +// it is (0: not a LibriVox recording, or a metaserve too old to say) and its +// Internet Archive item. +type LibrivoxRecording struct { + ID string + LibrivoxID int + ArchiveID string +} + +// LibrivoxCount is how many of the work's recordings are LibriVox projects. +func (w LibrivoxWork) LibrivoxCount() int { + n := 0 + for _, r := range w.Recordings { + if r.LibrivoxID > 0 { + n++ + } + } + return n +} + +// LibrivoxWork reads a work for a LibriVox fetch. ErrDisabled, ErrNotFound and +// ErrUnavailable as LibrivoxAudiobook's. +func (c *Client) LibrivoxWork(ctx context.Context, workID string) (LibrivoxWork, error) { + if !c.Enabled() { + return LibrivoxWork{}, ErrDisabled + } + var res struct { + ID string `json:"id"` + Title string `json:"title"` + Authors []struct { + Name string `json:"name"` + } `json:"authors"` + Recordings []struct { + ID string `json:"id"` + LibrivoxID int `json:"librivox_id"` + ArchiveID string `json:"archive_id"` + } `json:"recordings"` + } + found, ok := c.getJSON(ctx, "/api/v1/works/"+url.PathEscape(workID), &res) + switch { + case ctx.Err() != nil: + return LibrivoxWork{}, ctx.Err() + case !ok: + return LibrivoxWork{}, ErrUnavailable + case !found: + return LibrivoxWork{}, ErrNotFound + } + w := LibrivoxWork{ID: res.ID, Title: res.Title} + for _, a := range res.Authors { + if name := strings.TrimSpace(a.Name); name != "" { + w.Authors = append(w.Authors, name) + } + } + for _, r := range res.Recordings { + w.Recordings = append(w.Recordings, LibrivoxRecording{ID: r.ID, LibrivoxID: r.LibrivoxID, ArchiveID: r.ArchiveID}) + } + return w, nil +} diff --git a/internal/metaops/librivox_test.go b/internal/metaops/librivox_test.go new file mode 100644 index 0000000..a559044 --- /dev/null +++ b/internal/metaops/librivox_test.go @@ -0,0 +1,112 @@ +package metaops + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +// feed192 is meta's LibriVox-shaped answer for project 192 (extended=1, +// coverart=1), in LibriVox's own shape: string ids, numbers and playtimes. +const feed192 = `{"books":[{"id":"192","title":"Johnny Crow's Garden","description":"","url_text_source":"", +"language":"English","copyright_year":"1903","num_sections":"2","url_rss":"https://librivox.org/rss/192", +"url_zip_file":"https://archive.org/compress/johnny_crow_garden_librivox/x.zip","url_project":"", +"url_librivox":"https://librivox.org/johnny-crows-garden-by-l-leslie-brooke/","url_other":"", +"totaltime":"0:12:00","totaltimesecs":720, +"authors":[{"id":"404","first_name":"L. Leslie","last_name":"Brooke","dob":"1862","dod":"1940"}], +"coverart_jpg":"https://archive.org/download/johnny_crow_garden_librivox/johnnycrow_1104.jpg","coverart_pdf":null,"coverart_thumbnail":null, +"url_iarchive":"https://www.archive.org/details/johnny_crow_garden_librivox", +"sections":[{"id":"","section_number":"1","title":"Johnny Crow's Garden","listen_url":"https://www.archive.org/download/johnny_crow_garden_librivox/johnnycrow_01_brooke_64kb.mp3","language":"English","playtime":"300","file_name":null,"readers":[]}, +{"id":"","section_number":"2","title":"Johnny Crow's Party","listen_url":"https://www.archive.org/download/johnny_crow_garden_librivox/johnnycrow_02_brooke_64kb.mp3","language":"English","playtime":"420","file_name":null,"readers":[]}], +"genres":[],"translators":[]}]}` + +const work192 = `{"id":"johnny-crows-garden-brooke","title":"Johnny Crow's Garden","authors":[{"id":"l-leslie-brooke","name":"L. Leslie Brooke"}], +"language":"en","series":[],"recordings":[ +{"id":"johnny-crows-garden-brooke-librivox-192","narrators":[],"asin":[],"isbn":[],"chapter_count":2,"librivox_id":192,"archive_id":"johnny_crow_garden_librivox"}, +{"id":"johnny-crows-garden-brooke-audible","narrators":[],"asin":[],"isbn":[],"chapter_count":0}]}` + +func TestLibrivoxReads(t *testing.T) { + var queries []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + queries = append(queries, r.URL.String()) + switch { + case r.URL.Path == "/api/v1/librivox/audiobooks" && r.URL.Query().Get("id") == "192": + _, _ = w.Write([]byte(feed192)) + case r.URL.Path == "/api/v1/librivox/audiobooks" && r.URL.Query().Get("id") == "500": + http.Error(w, "boom", http.StatusInternalServerError) + case r.URL.Path == "/api/v1/librivox/audiobooks": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":"Audiobooks could not be found"}`)) + case r.URL.Path == "/api/v1/works/johnny-crows-garden-brooke": + _, _ = w.Write([]byte(work192)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + c := NewClient(srv.URL + "/") + ctx := context.Background() + + b, err := c.LibrivoxAudiobook(ctx, 192) + if err != nil { + t.Fatal(err) + } + if b.ID != "192" || b.Title != "Johnny Crow's Garden" || len(b.Sections) != 2 || b.CoverartJPG == nil || + b.Sections[1].ListenURL != "https://www.archive.org/download/johnny_crow_garden_librivox/johnnycrow_02_brooke_64kb.mp3" || + b.Sections[0].SectionNumber != "1" || b.Sections[1].Playtime != "420" { + t.Fatalf("%+v", b) + } + if b.Authors[0].Name() != "L. Leslie Brooke" || b.ArchiveID() != "johnny_crow_garden_librivox" { + t.Fatalf("author %q item %q", b.Authors[0].Name(), b.ArchiveID()) + } + if queries[0] != "/api/v1/librivox/audiobooks?coverart=1&extended=1&id=192" { + t.Fatalf("asked %s", queries[0]) + } + if _, err := c.LibrivoxAudiobook(ctx, 7); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing project: %v", err) + } + if _, err := c.LibrivoxAudiobook(ctx, 500); !errors.Is(err, ErrUnavailable) { + t.Fatalf("a 500: %v", err) + } + + w, err := c.LibrivoxWork(ctx, "johnny-crows-garden-brooke") + if err != nil { + t.Fatal(err) + } + if w.Title != "Johnny Crow's Garden" || len(w.Authors) != 1 || w.Authors[0] != "L. Leslie Brooke" || w.LibrivoxCount() != 1 || + w.Recordings[0].ArchiveID != "johnny_crow_garden_librivox" { + t.Fatalf("%+v", w) + } + if _, err := c.LibrivoxWork(ctx, "nope"); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing work: %v", err) + } + + off := NewClient("") + if _, err := off.LibrivoxAudiobook(ctx, 192); !errors.Is(err, ErrDisabled) { + t.Fatalf("disabled: %v", err) + } + if _, err := off.LibrivoxWork(ctx, "x"); !errors.Is(err, ErrDisabled) { + t.Fatalf("disabled: %v", err) + } + down := NewClient("http://127.0.0.1:1") + if _, err := down.LibrivoxAudiobook(ctx, 192); !errors.Is(err, ErrUnavailable) { + t.Fatalf("unreachable: %v", err) + } +} + +func TestLibrivoxArchiveID(t *testing.T) { + for raw, want := range map[string]string{ + "https://www.archive.org/details/x_librivox": "x_librivox", + "https://archive.org/details/x.y-z/files": "x.y-z", + "https://librivox.org/not-an-item/": "", + "": "", + "https://www.archive.org/details/": "", + "https://www.archive.org/details/a?b=c#d": "a", + } { + if got := (LibrivoxBook{URLIArchive: raw}).ArchiveID(); got != want { + t.Errorf("ArchiveID(%q) = %q, want %q", raw, got, want) + } + } +} From 8df3515e909de96f64cd8af84d68e0346ef06320 Mon Sep 17 00:00:00 2001 From: KodeStar Date: Fri, 13 Nov 2026 16:11:49 +0000 Subject: [PATCH 04/22] Add librivox.fetch: download a LibriVox recording from archive.org into a books library The job reads the project's manifest from the Bridge's own meta base, fetches each section over https from archive.org only (every redirect checked), stages them in a dot folder beside the destination with Range/If-Range resume, and places Author/Title/NN - Section.mp3, cover.jpg and .audiosilo/librivox.json by one no-replace rename before a rescan. The title's folder taken, it uses Title (LibriVox ) once, then fails exists. A cancel or failure removes the staging folder; a stop or lost claim keeps it. Its plan and placement marker live in /librivox so a failed rescan resumes there. Registered on the download lane, so the server sees the librivox feature; the page labels the kind. Part of LibriVox L6-1 (Add from LibriVox). --- CLAUDE.md | 23 +- PACKAGES.md | 78 ++- internal/app/app.go | 9 +- internal/app/app_test.go | 4 +- internal/app/librivox.go | 26 + internal/librivoxjob/download.go | 283 ++++++++++ internal/librivoxjob/fake_test.go | 331 +++++++++++ internal/librivoxjob/librivoxjob.go | 688 +++++++++++++++++++++++ internal/librivoxjob/librivoxjob_test.go | 669 ++++++++++++++++++++++ internal/librivoxjob/names.go | 161 ++++++ internal/librivoxjob/names_test.go | 168 ++++++ internal/librivoxjob/state.go | 112 ++++ web/src/lib/generate.test.ts | 1 + web/src/lib/generate.ts | 1 + 14 files changed, 2544 insertions(+), 10 deletions(-) create mode 100644 internal/app/librivox.go create mode 100644 internal/librivoxjob/download.go create mode 100644 internal/librivoxjob/fake_test.go create mode 100644 internal/librivoxjob/librivoxjob.go create mode 100644 internal/librivoxjob/librivoxjob_test.go create mode 100644 internal/librivoxjob/names.go create mode 100644 internal/librivoxjob/names_test.go create mode 100644 internal/librivoxjob/state.go diff --git a/CLAUDE.md b/CLAUDE.md index dd113b1..1af7ed1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -157,6 +157,11 @@ internal/ -> rescan -> ASIN enrich audiblesync/ Phase 4c: audible.sync (lastpositions -> furthest-only events) + the push loop (feed -> SetLastPosition) importjob/ BookMatcher (link.LibraryBook) + Plan: match and place against a library + -- LibriVox L6 -- + librivoxjob/ librivox.fetch (download lane): the manifest from meta's LibriVox-shaped + feed, the sections from archive.org only (https, every redirect checked) + into a staging folder, Author/Title/NN - Section.mp3 + cover.jpg + + .audiosilo/librivox.json by one no-replace rename, rescan transfer/ Local.Place: SafeJoin + os.Root, free-space check, no-replace install placement/ source/ progress/ (the manager's; source trimmed to Record in 4b) web/ the page: Vite + React 19 + TS + Tailwind v4; pure logic in src/lib @@ -166,7 +171,7 @@ Dockerfile slim | cuda | agent | full; compose.example.yaml: server, bridge, **Dependency direction:** `app -> {api, jobs, link, mapping, verify, generate, agentbox, workers, accounts, setup, correct, store, secrets, config, auth, logring, -web, netguard, podcast, podcastjob, audibleacct, audiblejob, audiblesync}` plus the engine it wires; `accounts -> {agentproto, ghauth, secrets, store}`; `correct -> +web, netguard, podcast, podcastjob, audibleacct, audiblejob, audiblesync, librivoxjob}` plus the engine it wires; `accounts -> {agentproto, ghauth, secrets, store}`; `correct -> {contrib, jobs, metaops}`; `setup -> link`; `api` calls the injected Backend and the injected `auth.Manager` (login, sessions, the password), with types from jobs/link/mapping/store/logring; `verify -> {jobs, @@ -177,7 +182,8 @@ through its own Engine/Server interfaces; `podcastjob -> {jobs, link, mapping, n imports `audible`); `importjob -> {link, placement, source, audiosilo-server/pkg/match}`; `audibleacct -> {audible, link, secrets, source, store}`; `audiblejob -> {audible, audibleacct, fsutil, importjob, jobs, link, mapping, placement, -source, store, transfer}` (app wires both); `audiblesync -> {audible, audibleacct, jobs, link, store}`; `transfer -> {fsutil, mapping, progress}`; +source, store, transfer}` (app wires both); `librivoxjob -> {fsutil, importjob, jobs, link, +mapping, metaops, netguard, placement, transfer}`; `audiblesync -> {audible, audibleacct, jobs, link, store}`; `transfer -> {fsutil, mapping, progress}`; `jobs -> {link, store}`; `mapping -> {fsutil, link}`; `link`, `agentproto` and `workerproto` are leaves; `workers -> {asr, fsutil, scheduler, state, store, transcript, workerproto}`; `worker -> {asr, fsutil, workerproto}` @@ -205,7 +211,10 @@ in PACKAGES.md. book writer is `generate` (its three jobs): `characters.json`, `recaps.json`, `description.json` (Phase 1c), `chapters.json` (Phase 1d), `readalong.json` (Phase 3c-1) and `bridge.json` in the book's `.audiosilo/` (a single-file book's prefixed - `.`). Podcast writers + `.`). `librivox.fetch` (LibriVox L6) writes whole new book folders: its own + dot staging folder beside the destination (removed outright on a cancel or a + failure: it holds only the job's own downloads), then one no-replace rename, + never into an existing book. Podcast writers require a podcast-kind library and its checked marker mapping (a job claimed before the check waits for it); they write only the root marker, show folders, feed.json and sniffed JPEG/PNG artwork. Book writers refuse podcast libraries. @@ -440,6 +449,14 @@ key set) with Range, a cut and an expiry. No migration. server's still places back to Audible; a refused account neither syncs nor pushes, a sync error never refuses one; the page's sync switch went (the console owns it). +**LibriVox L6-1 (Add from LibriVox, Bridge side):** `librivox.fetch` +(`librivoxjob`, LIBRIVOX-PLAN.md L6): the manifest from the Bridge's own meta base, +the sections from archive.org only (`AllowedURL` on every URL and redirect through +netguard's new per-request `Allow`), staged in the library and placed by one +no-replace rename with `cover.jpg` and `.audiosilo/librivox.json`, then rescanned; +resumable per section, its plan and placement marker in `/librivox/`. The +server records the book's LibriVox identity from the job's result (its fold). + See BRIDGE-PLAN.md's Status table for what is next. Operative caveats carried from sidecars (PACKAGES.md has the detail): diff --git a/PACKAGES.md b/PACKAGES.md index 6105e54..82af066 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -1734,7 +1734,12 @@ internal/ landing in between is never cleared nor parked over. metaops/ meta.audiosilo.app client (coverage/lookup, capped 1h TTL caches, graceful degrade) + the PathAllowed/AllowedPath containment checks - (Phase 1a passes the mapped mounts as the roots). glossary.go adds + (Phase 1a passes the mapped mounts as the roots). librivox.go (LibriVox + L6): LibrivoxAudiobook (the LibriVox-shaped feed, one project with + extended=1&coverart=1; LibriVox's string ids, nullable coverart_jpg, + ArchiveID from url_iarchive) and LibrivoxWork (works/{id}: title, + author names, each recording's librivox_id and archive_id), uncached, + with ErrDisabled / ErrNotFound / ErrUnavailable apart. glossary.go adds SeriesGlossary (work -> its series -> the OTHER volumes' contributed character names/aliases, <= 12 siblings, own TTL caches), the canonical spelling evidence the spelling stage checks a transcript against; it @@ -2036,6 +2041,10 @@ internal/ failures, respecting Retry-After. Bad gzip data (empty, header, checksum, stream) fails at once as not_a_feed; a read the connection broke is retried. Errors never contain remote text or URLs. + LibriVox L6: Request and StreamRequest take an optional Allow, asked + about the first URL and every redirect's (CheckRedirect); a refusal is + blocked_host before the refused address is dialled. Nil keeps the + old rule (the podcasts pass none). podcast/ pure feed.json v1 types, rooted regular-file IO with capped atomic writes, URL normalization/hash/privacy/redaction, NFC show names, stable episode paths, identity merge and adaptive polling. Spike 5 @@ -2589,12 +2598,74 @@ internal/ the library holds a record is BookMatcher's). Run, the existence check and ApplyDestinations went in Phase 4b: audiblejob's import runs the steps. The manager's serverapi client went: the Bridge holds - the paged listing. Deps: link, placement, source, + the paged listing. AllBooks (books.go, LibriVox L6) reads that listing + whole for audiblejob and librivoxjob. Deps: link, placement, source, audiosilo-server/pkg/match. placement/ where a new book goes (the manager's): Auto (series siblings' folder and shortcode convention, else author/title) and templates. Phase 4b: a generated segment loses its leading dots too ("...And Ladies of the Club", ".hack"): a hidden name is never indexed. + LibriVox L6 exports Sanitize (librivoxjob names its section files + with it). + librivoxjob/ LibriVox L6 (LIBRIVOX-PLAN.md), librivox.fetch on the download lane + (capacity 1, shared with audible.import: one book at a time). Params + {librivox_id, work_id, recording_id, library_id} (server-built, + strict; unknown fields ignored). Manifest from the Bridge's OWN meta + base (metaops.LibrivoxAudiobook: /api/v1/librivox/audiobooks?id=N& + extended=1&coverart=1, LibriVox's shape; decision 5, the server's + mirror is never asked) and the work (metaops.LibrivoxWork, works/{id}; + a 404 there is no failure, the project names the folder then). Every + section URL must be https on archive.org or *.archive.org + (AllowedURL, checked before anything is fetched or written, and on + every redirect through netguard's per-request Allow; never + librivox.org), fetched by the app's own netguard client (public + addresses only, not the podcasts' allow_networks). Placement: + placement.Suggest standalone (an existing author folder from + importjob.AuthorFolders reused verbatim, else the sanitized author), + NOT importjob.Planner.Plan: its match.CleanTitle drops "(version 2)" + and "Vol. 001", which tell two projects of one work apart. Title = + the work's, or the project's own when the work has more than one + LibriVox recording; author = the first credit, Various over three; + no author places at the library root; nothing left of a title is + "LibriVox ". Names: control characters and space runs folded, + Sanitize, cut to 180 bytes on a rune boundary; files "NN -
." (NN the 1-based position, at least two digits, the + count's width; never section_number; an untitled section is + "Section N"; the URL's audio extension, else .mp3). The title's + folder taken, once " (LibriVox <id>)", else exists; a folder + already holding this project's librivox.json is the book (rescan + only); an author folder that is a listed book or holds audio is + exists. Staging <parent>/.audiosilo-librivox-<id>/ (a dot folder the + scan skips), every write through the mount's os.Root (0755/0644): + each section to <name>.part, its strong ETag and size recorded in + the state before the body, a resume asks Range/If-Range + (netguard.ResumeAction proves the 206; anything unproven starts the + file again), renamed to its name only at the stated size (a finished + file is skipped on resume); three attempts in a row that add nothing + fail archive_unavailable, as does a 4xx; the library's free space + 1.1x each file's rest, else disk_full. cover.jpg best effort (only a + sniffed JPEG from archive.org), then .audiosilo/librivox.json + {version 1, librivox_id, work_id, recording_id, archive_id} (decision + 3), the mapping checked again (a resumed or late job parks + not_writable), ONE fsutil.RenameNoReplace staging -> book folder (a + folder that raced in takes the fallback once, never replaced), then + the rescan (3 tries). State <data>/librivox/<id>.json (state.go: the + library, parent and whether the fetch made it, staging, destination, + the partials' validators, then placed + count + bytes): a resumed run + follows it rather than planning again, Abandon finds the staging + folder by it, and once placed it is the placement marker (a failed + rescan keeps the book and the state; the retry goes on from the + rescan). A stop or a lost claim keeps staging and state; a cancel, a + failure or Abandon (not on a lost claim) removes the staging folder, + the author folder it made when empty, and the state. A lost claim + never handed out again leaves its staging folder (hidden, unindexed). + Steps manifest, download (done/total = sections), cover, place, + rescan. Result {library_id, path (the rescan's), librivox_id, files, + bytes} (files and bytes: the sections and the cover); a failure is + Failure {code, message} as result and error, codes not_writable, + exists, disk_full, not_found, archive_unavailable, metadata_off, + never a URL. Deps: fsutil, importjob, jobs, link, mapping, metaops, + netguard, placement, transfer, audiosilo-server/pkg/match. source/ Record (normalized metadata of one importable book: the ASIN, UnavailableReason; Phase 4a: Authors (every credit), SeriesPosition as the source spells it, ReleaseDate YYYY-MM-DD or "") (source.go only; @@ -3553,5 +3624,6 @@ and the shared clip-floor predicate `qa.ClipStartInRange`/`qa.ClipStartFloorSec` `state` is pure. Phase 4a: `audibleacct -> {audible, link, secrets, source, store}` and `audiblejob -> {audible, audibleacct, fsutil, importjob, jobs, link, mapping, placement, source, store, transfer}` (Phase 4b's import; app wires both; -neither imports app or api). Handlers marshal DTOs and call into the injected packages; they +neither imports app or api); LibriVox L6: `librivoxjob -> {fsutil, importjob, jobs, link, +mapping, metaops, netguard, placement, transfer}` (app wires it). Handlers marshal DTOs and call into the injected packages; they hold no logic (state transitions live in `state`, dispatch in `scheduler`). diff --git a/internal/app/app.go b/internal/app/app.go index 3340a53..8493daf 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -75,6 +75,9 @@ type Bridge struct { runner *jobs.Runner auth *auth.Manager podcastHTTP *netguard.Client + // librivoxHTTP is librivox.fetch's guarded client: public addresses only + // (never the podcasts' allow_networks), archive.org hosts only per request. + librivoxHTTP *netguard.Client // box is the agent container runner (decision 15), gen the sidecars.generate // job, engine the sidecars engine once it has started. box *agentbox.Runner @@ -216,7 +219,8 @@ func New(ctx context.Context, opts Options) (*Bridge, string, error) { } return nil }}) - corrector := correct.New(correct.Options{Meta: metaops.NewClient(cfg.Metadata.BaseURL), Tokens: b.tokens, + meta := metaops.NewClient(cfg.Metadata.BaseURL) + corrector := correct.New(correct.Options{Meta: meta, Tokens: b.tokens, BaseURL: cfg.Contribution.APIBaseURL, CoreRepo: cfg.Contribution.CoreRepo, KV: db, ServerID: serverID, Log: opts.Logger}) var audiblePoll *audiblejob.Handler var audibleImport *audiblejob.Importer @@ -225,7 +229,7 @@ func New(ctx context.Context, opts Options) (*Bridge, string, error) { audibleSync, b.audiblePush = b.newAudibleSync(db, opts.Logger, serverID) b.runner = jobs.New(db, b.server, b.tokenRefused, jobs.Options{Log: opts.Logger, ServerID: serverID}, verifier, b.gen, b.gen.DescribeJob(), b.gen.ChaptersJob(), b.gen.ReadalongJob(), b.gen.ContributeJob(), corrector, podcasts, podcasts.PollJob(), podcasts.FetchJob(), podcasts.RetainJob(), - audiblePoll, audibleImport, audibleSync) + audiblePoll, audibleImport, audibleSync, b.newLibrivox(meta, opts.Logger)) conn.onChange = b.linkChanged return b, oneTime, nil } @@ -298,6 +302,7 @@ var listen = func(ctx context.Context, addr string) (net.Listener, error) { // or the page's server fails (its error is returned once everything stopped). func (b *Bridge) Run(ctx context.Context) error { defer b.podcastHTTP.Close() + defer b.librivoxHTTP.Close() defer func() { _ = b.db.Close() }() // The heartbeat and the runner stop with ctx: a failing page server must // stop them too, or Run would wait for them for ever. diff --git a/internal/app/app_test.go b/internal/app/app_test.go index aeebf01..02447bb 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -144,7 +144,7 @@ func TestPairBeatMapAndRevoke(t *testing.T) { return ok && len(st.Libraries) == 1 && st.Libraries[0].Mapped && st.Libraries[0].Writable }) st, _ := fs.lastBeat() - if st.Version != "0.1.0" || strings.Join(st.Kinds, ",") != "audible.import,audible.poll,audible.sync,bridge.verify,chapters.detect,meta.correct,podcast.fetch,podcast.poll,podcast.retain,podcast.subscribe,readalong.align,sidecars.contribute,sidecars.describe,sidecars.generate" || st.UIURL != "http://bridge.lan:8091" { + if st.Version != "0.1.0" || strings.Join(st.Kinds, ",") != "audible.import,audible.poll,audible.sync,bridge.verify,chapters.detect,librivox.fetch,meta.correct,podcast.fetch,podcast.poll,podcast.retain,podcast.subscribe,readalong.align,sidecars.contribute,sidecars.describe,sidecars.generate" || st.UIURL != "http://bridge.lan:8091" { t.Fatalf("heartbeat = %+v", st) } // Once the engine runs, every heartbeat carries the estimates. @@ -283,7 +283,7 @@ func TestEngineViewAndJobRoutes(t *testing.T) { if _, err := b.Job(context.Background(), 5); !errors.Is(err, api.ErrNotFound) { t.Fatalf("an unknown job = %v", err) } - if k := strings.Join(b.runner.Kinds(), ","); k != "audible.import,audible.poll,audible.sync,bridge.verify,chapters.detect,meta.correct,podcast.fetch,podcast.poll,podcast.retain,podcast.subscribe,readalong.align,sidecars.contribute,sidecars.describe,sidecars.generate" { + if k := strings.Join(b.runner.Kinds(), ","); k != "audible.import,audible.poll,audible.sync,bridge.verify,chapters.detect,librivox.fetch,meta.correct,podcast.fetch,podcast.poll,podcast.retain,podcast.subscribe,readalong.align,sidecars.contribute,sidecars.describe,sidecars.generate" { t.Fatalf("kinds = %s", k) } } diff --git a/internal/app/librivox.go b/internal/app/librivox.go new file mode 100644 index 0000000..329600e --- /dev/null +++ b/internal/app/librivox.go @@ -0,0 +1,26 @@ +package app + +import ( + "log/slog" + "path/filepath" + + "github.com/kodestar/audiosilo-bridge/internal/librivoxjob" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/netguard" +) + +// newLibrivox builds librivox.fetch (LIBRIVOX-PLAN.md L6): the manifest from +// the Bridge's own metadata base (decision 5), the files from archive.org +// through a guarded client of its own (decision 7), placed in a mapped books +// library and rescanned. Its state lives in <data>/librivox. +func (b *Bridge) newLibrivox(meta *metaops.Client, log *slog.Logger) *librivoxjob.Handler { + b.librivoxHTTP = netguard.New(netguard.Options{Version: b.version}) + return librivoxjob.New(librivoxjob.Options{Meta: meta, Fetcher: b.librivoxHTTP, Libraries: b.libs, Mount: b.mount, + StateDir: filepath.Join(b.dataDir, "librivox"), Log: log, + Server: func() librivoxjob.Server { + if cl := b.conn.Client(); cl != nil { + return cl + } + return nil + }}) +} diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go new file mode 100644 index 0000000..a100051 --- /dev/null +++ b/internal/librivoxjob/download.go @@ -0,0 +1,283 @@ +package librivoxjob + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "math" + "net/http" + "os" + "path" + "path/filepath" + "time" + + "github.com/kodestar/audiosilo-bridge/internal/fsutil" + "github.com/kodestar/audiosilo-bridge/internal/jobs" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/netguard" + "github.com/kodestar/audiosilo-bridge/internal/transfer" +) + +// downloadAttempts bounds a section's attempts in a row that add nothing to +// its partial (a dropped connection resumes from it each time; an attempt that +// added to it starts the count again). +const downloadAttempts = 3 + +// maxFileBytes caps one section's file (a LibriVox section is tens of MB). +const maxFileBytes int64 = 2 << 30 + +// maxCoverBytes caps the cover. +const maxCoverBytes int64 = 20 << 20 + +// maxRetryAfter bounds how long a Retry-After is waited out. +const maxRetryAfter = 5 * time.Minute + +// partSuffix names a section's partial download in the staging folder. +const partSuffix = ".part" + +// download fetches every section not in the staging folder yet: a file there +// under its own name is finished (it is renamed from its .part only once +// whole), so a resumed run skips it and goes on with the next, resuming that +// one's .part with Range/If-Range when its strong ETag was recorded. +func (h *Handler) download(ctx context.Context, f *fetch, sections []metaops.LibrivoxSection, names []string) error { + root, err := os.OpenRoot(f.mount) + if err != nil { + return fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + } + defer func() { _ = root.Close() }() + n := len(sections) + for i, s := range sections { + h.report(f.rep, jobs.Progress{Step: StepDownload, Message: fmt.Sprintf("Downloading section %d of %d from archive.org", i+1, n), + Done: i, Total: n}) + if st, err := root.Lstat(path.Join(f.st.Staging, names[i])); err == nil && st.Mode().IsRegular() { + continue + } + if err := h.section(ctx, root, f, i, s.ListenURL, names[i]); err != nil { + return err + } + } + h.report(f.rep, jobs.Progress{Step: StepDownload, Message: "Downloaded from archive.org", Done: n, Total: n}) + return nil +} + +// retry is an attempt that may be tried again: after RetryAfter when the +// server said so, and whether it added to the partial. +type retry struct { + retryAfter int + added bool + cause string +} + +func (r *retry) Error() string { return r.cause } + +// section downloads one section to <staging>/<name>, trying again while +// attempts add to the partial (downloadAttempts in a row that add nothing +// fail archive_unavailable). +func (h *Handler) section(ctx context.Context, root *os.Root, f *fetch, i int, rawURL, name string) error { + failures := 0 + for { + err := h.attempt(ctx, root, f, i, rawURL, name) + var r *retry + switch { + case err == nil: + return nil + case ctx.Err() != nil: + return ctx.Err() + case !errors.As(err, &r): + return err + } + if r.added { + failures = 0 + } + if failures++; failures >= downloadAttempts { + return fail(CodeArchiveUnavailable, "archive.org didn't deliver section %d of LibriVox recording %d (%s): try again later.", + i+1, f.p.LibrivoxID, r.cause) + } + h.o.Log.Info("librivox: a section's download failed; trying again", "librivox_id", f.p.LibrivoxID, "section", i+1, + "attempt", failures, "cause", r.cause) + wait := max(time.Duration(failures*failures)*2*time.Second, min(time.Duration(r.retryAfter)*time.Second, maxRetryAfter)) + if werr := h.o.Wait(ctx, wait); werr != nil { + return werr + } + } +} + +// attempt is one GET of a section: a 200 writes the file from its start, a +// 206 proven to continue the partial (netguard.ResumeAction) appends to it, +// anything unproven starts again. The file is renamed from its .part once it +// holds the size the response stated. +func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, rawURL, name string) error { + final := path.Join(f.st.Staging, name) + part := final + partSuffix + offset := int64(0) + if st, err := root.Lstat(part); err == nil && st.Mode().IsRegular() { + offset = st.Size() + } + prev := f.st.Files[name] + if offset == 0 { + prev = partState{} + } + resp, err := h.o.Fetcher.Stream(ctx, netguard.StreamRequest{URL: rawURL, Offset: offset, ETag: prev.ETag, Allow: AllowedURL}) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + e := netguard.Classify(err) + if retryableCode(e.Code) { + return &retry{retryAfter: e.RetryAfterSeconds, cause: e.Code} + } + return fail(CodeArchiveUnavailable, "The Bridge can't download section %d of LibriVox recording %d from archive.org (%s).", + i+1, f.p.LibrivoxID, e.Code) + } + defer func() { _ = resp.Body.Close() }() + + appending := false + total := int64(0) + switch { + case resp.Encoding != "" && resp.Encoding != "identity": + return fail(CodeArchiveUnavailable, "archive.org sent section %d of LibriVox recording %d encoded (%s).", i+1, f.p.LibrivoxID, resp.Encoding) + case resp.Status == http.StatusOK: + total = max(resp.ContentLength, 0) + case resp.Status == http.StatusPartialContent && netguard.ResumeAction(offset, prev.Total, prev.ETag, resp) == "append": + appending, total = true, prev.Total + case resp.Status == http.StatusPartialContent, resp.Status == http.StatusRequestedRangeNotSatisfiable: + // Not a continuation the Bridge can prove: the partial goes and the + // next attempt asks for the whole file. + _ = root.Remove(part) + delete(f.st.Files, name) + return &retry{cause: fmt.Sprintf("http_%d", resp.Status)} + case resp.Status == http.StatusRequestTimeout, resp.Status == http.StatusTooManyRequests, resp.Status >= 500: + return &retry{retryAfter: resp.RetryAfterSeconds, cause: fmt.Sprintf("http_%d", resp.Status)} + default: + return fail(CodeArchiveUnavailable, "archive.org answered HTTP %d for section %d of LibriVox recording %d.", resp.Status, i+1, f.p.LibrivoxID) + } + if total > maxFileBytes { + return fmt.Errorf("section %d is larger than the Bridge's cap for one file", i+1) + } + start := int64(0) + if appending { + start = offset + } + if total > 0 { + need := int64(math.Ceil(transfer.DefaultHeadroom * float64(total-start))) + if free, err := h.o.Free(filepath.Join(f.mount, filepath.FromSlash(f.st.Staging))); err == nil && free < need { + return fail(CodeDiskFull, "The library is short of space for section %d of the recording.", i+1) + } + } + // The validators go into the state before the body: a stop during the + // body resumes this file from what it holds. + if f.st.Files == nil { + f.st.Files = map[string]partState{} + } + ps := partState{Total: total} + if netguard.StrongETag(resp.ETag) { + ps.ETag = resp.ETag + } + f.st.Files[name] = ps + if err := h.saveState(f.p.LibrivoxID, f.st); err != nil { + return fmt.Errorf("recording a download's progress in the Bridge's data: %w", err) + } + + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC + if appending { + flags = os.O_WRONLY | os.O_CREATE | os.O_APPEND + } + file, err := root.OpenFile(part, flags, 0o644) + if err != nil { + return writeFailure(err, part) + } + w := &writeErr{w: file} + n, cerr := io.Copy(w, io.LimitReader(resp.Body, maxFileBytes-start+1)) + if err := file.Sync(); err != nil && w.err == nil { + w.err = err + } + if err := file.Close(); err != nil && w.err == nil { + w.err = err + } + size := start + n + switch { + case w.err != nil: + return writeFailure(w.err, part) + case size > maxFileBytes: + _ = root.Remove(part) + return fmt.Errorf("section %d is larger than the Bridge's cap for one file", i+1) + case cerr != nil: + return &retry{added: n > 0, cause: "network"} + case total > 0 && size != total: + if size > total { + _ = root.Remove(part) + delete(f.st.Files, name) + } + return &retry{added: n > 0, cause: "short"} + } + if err := root.Rename(part, final); err != nil { + return writeFailure(err, final) + } + delete(f.st.Files, name) + if err := h.saveState(f.p.LibrivoxID, f.st); err != nil { + h.o.Log.Warn("librivox: recording a finished section failed", "librivox_id", f.p.LibrivoxID, "err", err) + } + return nil +} + +// retryableCode reports whether a netguard failure is worth another attempt +// (a name lookup, a timeout, a dropped connection); a blocked address or host, +// an invalid URL, a TLS failure or a redirect loop is not. +func retryableCode(code string) bool { + return code == "dns" || code == "timeout" || code == "network" +} + +// writeErr remembers a write's failure, so a failed copy tells the library's +// fault (disk_full, not_writable) from the connection's (try again). +type writeErr struct { + w io.Writer + err error +} + +func (w *writeErr) Write(p []byte) (int, error) { + n, err := w.w.Write(p) + if err != nil { + w.err = err + } + return n, err +} + +// cover fetches the project's cover into the staging folder as cover.jpg: +// best effort (a book without one is still a book), only from archive.org, +// and only when it is a JPEG. +func (h *Handler) cover(ctx context.Context, f *fetch, book metaops.LibrivoxBook) { + if book.CoverartJPG == nil || *book.CoverartJPG == "" { + return + } + root, err := os.OpenRoot(f.mount) + if err != nil { + return + } + defer func() { _ = root.Close() }() + if _, err := root.Lstat(path.Join(f.st.Staging, coverName)); err == nil { + return + } + h.report(f.rep, jobs.Progress{Step: StepCover, Message: "Downloading the cover from archive.org"}) + log := h.o.Log.With("librivox_id", f.p.LibrivoxID) + if !allowedRaw(*book.CoverartJPG) { + log.Info("librivox: the cover isn't on archive.org; the book goes without one") + return + } + resp, err := h.o.Fetcher.Fetch(ctx, netguard.Request{URL: *book.CoverartJPG, MaxBytes: maxCoverBytes, Allow: AllowedURL}) + switch { + case err != nil: + log.Info("librivox: the cover couldn't be downloaded; the book goes without one", "cause", netguard.Classify(err).Code) + return + case resp.Status != http.StatusOK || http.DetectContentType(resp.Body) != "image/jpeg": + log.Info("librivox: the cover isn't a JPEG; the book goes without one") + return + } + if err := fsutil.WriteRootAtomic(root, f.st.Staging, coverName, resp.Body, 0o644, nil); err != nil && !errors.Is(err, fs.ErrExist) { + log.Warn("librivox: writing the cover failed; the book goes without one", "err", err) + } +} + +// coverName is the cover's file in the book's folder. +const coverName = "cover.jpg" diff --git a/internal/librivoxjob/fake_test.go b/internal/librivoxjob/fake_test.go new file mode 100644 index 0000000..97b65b0 --- /dev/null +++ b/internal/librivoxjob/fake_test.go @@ -0,0 +1,331 @@ +package librivoxjob + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + "github.com/kodestar/audiosilo-bridge/internal/jobs" + "github.com/kodestar/audiosilo-bridge/internal/link" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/netguard" +) + +// fakeMeta answers the manifest and the work. +type fakeMeta struct { + book metaops.LibrivoxBook + bookErr error + work metaops.LibrivoxWork + workErr error + reads int +} + +func (m *fakeMeta) LibrivoxAudiobook(_ context.Context, id int) (metaops.LibrivoxBook, error) { + m.reads++ + if m.bookErr != nil { + return metaops.LibrivoxBook{}, m.bookErr + } + if m.book.ID != strconv.Itoa(id) { + return metaops.LibrivoxBook{}, metaops.ErrNotFound + } + return m.book, nil +} + +func (m *fakeMeta) LibrivoxWork(context.Context, string) (metaops.LibrivoxWork, error) { + return m.work, m.workErr +} + +// remote is one file the fake archive.org serves. +type remote struct { + body []byte + etag string + status int // an error status instead of the file + // redirect: the file is served from this URL, which Allow is asked about + // first (netguard's CheckRedirect). + redirect string + // cut: each GET pops one; >= 0 delivers that many bytes of the answer and + // then fails the read (a dropped connection). + cut []int + // onCut runs when a cut fires (a test stops the job there). + onCut func() +} + +// request is one GET the fake saw. +type request struct { + url string + offset int64 + etag string +} + +// fakeFetcher is archive.org: Stream with strong-ETag ranges, Fetch for the +// cover, each URL and redirect through the request's Allow. +type fakeFetcher struct { + mu sync.Mutex + files map[string]*remote + requests []request +} + +func (f *fakeFetcher) seen() []request { + f.mu.Lock() + defer f.mu.Unlock() + return append([]request(nil), f.requests...) +} + +func (f *fakeFetcher) resolve(raw string, allow func(*url.URL) bool) (*remote, error) { + for _, at := range []string{raw} { + u, err := url.Parse(at) + if err != nil || (allow != nil && !allow(u)) { + return nil, &netguard.Error{Code: "blocked_host", Message: "blocked"} + } + } + r := f.files[raw] + if r == nil { + return &remote{status: http.StatusNotFound}, nil + } + if r.redirect != "" { + u, err := url.Parse(r.redirect) + if err != nil || (allow != nil && !allow(u)) { + return nil, &netguard.Error{Code: "blocked_host", Message: "blocked"} + } + } + return r, nil +} + +type cutReader struct { + r io.Reader + left int + onCut func() +} + +func (c *cutReader) Read(p []byte) (int, error) { + if c.left <= 0 { + if c.onCut != nil { + c.onCut() + } + return 0, errors.New("connection reset") + } + if len(p) > c.left { + p = p[:c.left] + } + n, err := c.r.Read(p) + c.left -= n + return n, err +} + +func (f *fakeFetcher) Stream(_ context.Context, in netguard.StreamRequest) (*netguard.StreamResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.requests = append(f.requests, request{url: in.URL, offset: in.Offset, etag: in.ETag}) + r, err := f.resolve(in.URL, in.Allow) + if err != nil { + return nil, err + } + if r.status != 0 { + return &netguard.StreamResponse{Status: r.status, Body: io.NopCloser(bytes.NewReader(nil)), ContentLength: 0}, nil + } + resp := &netguard.StreamResponse{Status: http.StatusOK, ETag: r.etag, ContentLength: int64(len(r.body))} + body := r.body + // Range only with a strong ETag, as netguard sends it. + if in.Offset > 0 && netguard.StrongETag(in.ETag) && in.ETag == r.etag && in.Offset < int64(len(r.body)) { + resp.Status = http.StatusPartialContent + body = r.body[in.Offset:] + resp.ContentLength = int64(len(body)) + resp.ContentRange = fmt.Sprintf("bytes %d-%d/%d", in.Offset, len(r.body)-1, len(r.body)) + } + var rd io.Reader = bytes.NewReader(body) + if len(r.cut) > 0 { + n := r.cut[0] + r.cut = r.cut[1:] + if n >= 0 { + rd = &cutReader{r: rd, left: n, onCut: r.onCut} + } + } + resp.Body = io.NopCloser(rd) + return resp, nil +} + +func (f *fakeFetcher) Fetch(_ context.Context, in netguard.Request) (*netguard.Response, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.requests = append(f.requests, request{url: in.URL}) + r, err := f.resolve(in.URL, in.Allow) + if err != nil { + return nil, err + } + if r.status != 0 { + return nil, &netguard.Error{Code: "http_" + strconv.Itoa(r.status), Message: "status"} + } + return &netguard.Response{Status: http.StatusOK, Body: r.body}, nil +} + +// fakeServer is the server link: the library's books and the rescan. +type fakeServer struct { + mu sync.Mutex + books []link.LibraryBook + rescanErr []error + rescanned []string + answer string // the rescan's book path ("" = the path asked) +} + +func (s *fakeServer) LibraryBooks(context.Context, int64, string, int) (*link.LibraryBooks, error) { + return &link.LibraryBooks{Books: s.books}, nil +} + +func (s *fakeServer) Rescan(_ context.Context, _ int64, p string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.rescanned = append(s.rescanned, p) + if len(s.rescanErr) > 0 { + err := s.rescanErr[0] + s.rescanErr = s.rescanErr[1:] + if err != nil { + return "", err + } + } + if s.answer != "" { + return s.answer, nil + } + return p, nil +} + +// fakeLibs is the mapping registry: library 3, books, mapped at mount. +type fakeLibs struct { + mount string + writable bool + kind string +} + +func (l *fakeLibs) Library(id int64) (link.Library, bool) { + if id != 3 { + return link.Library{}, false + } + return link.Library{ID: 3, Name: "Books", Root: "/srv/books", Kind: l.kind}, true +} + +func (l *fakeLibs) Status(id int64) (link.LibraryStatus, bool) { + if id != 3 { + return link.LibraryStatus{}, false + } + return link.LibraryStatus{LibraryID: 3, MountPath: l.mount, Mapped: true, Writable: l.writable}, true +} + +// harness is a handler over the fakes, a library folder and a state folder. +type harness struct { + h *Handler + meta *fakeMeta + fetch *fakeFetcher + srv *fakeServer + libs *fakeLibs + mount string + stateDir string + free int64 + progress []jobs.Progress + mu sync.Mutex +} + +const ( + testID = 192 + testWork = "johnny-crows-garden-brooke" + testRec = "johnny-crows-garden-brooke-librivox-192" + testItem = "johnny_crow_garden_librivox" + testAuthor = "L. Leslie Brooke" + testTitle = "Johnny Crow's Garden" +) + +func sectionURL(n int) string { + return fmt.Sprintf("https://www.archive.org/download/%s/johnnycrow_%02d_brooke_64kb.mp3", testItem, n) +} + +const coverURL = "https://archive.org/download/" + testItem + "/johnnycrow_1104.jpg" + +// jpeg is the smallest prefix http.DetectContentType calls image/jpeg. +var jpeg = []byte("\xFF\xD8\xFFcover-bytes") + +func newHarness(t *testing.T) *harness { + t.Helper() + x := &harness{mount: t.TempDir(), stateDir: filepath.Join(t.TempDir(), "librivox"), free: 1 << 40} + cover := coverURL + x.meta = &fakeMeta{ + book: metaops.LibrivoxBook{ID: strconv.Itoa(testID), Title: testTitle, + Authors: []metaops.LibrivoxPerson{{FirstName: "L. Leslie", LastName: "Brooke"}}, + URLIArchive: "https://www.archive.org/details/" + testItem, CoverartJPG: &cover, + Sections: []metaops.LibrivoxSection{ + {SectionNumber: "1", Title: "Johnny Crow's Garden", ListenURL: sectionURL(1)}, + {SectionNumber: "2", Title: "Johnny Crow's Party", ListenURL: sectionURL(2)}, + {SectionNumber: "3", Title: "Johnny Crow's New Garden", ListenURL: sectionURL(3)}, + }}, + work: metaops.LibrivoxWork{ID: testWork, Title: testTitle, Authors: []string{testAuthor}, + Recordings: []metaops.LibrivoxRecording{{ID: testRec, LibrivoxID: testID, ArchiveID: testItem}}}, + } + x.fetch = &fakeFetcher{files: map[string]*remote{ + sectionURL(1): {body: bytes.Repeat([]byte("1"), 1000), etag: `"etag-1"`}, + sectionURL(2): {body: bytes.Repeat([]byte("2"), 2000), etag: `"etag-2"`}, + sectionURL(3): {body: bytes.Repeat([]byte("3"), 3000), etag: `"etag-3"`}, + coverURL: {body: jpeg}, + }} + x.srv = &fakeServer{} + x.libs = &fakeLibs{mount: x.mount, writable: true} + x.h = New(Options{Meta: x.meta, Fetcher: x.fetch, Libraries: x.libs, StateDir: x.stateDir, + Mount: func(int64, string) (string, bool) { return x.mount, true }, + Server: func() Server { return x.srv }, + Free: func(string) (int64, error) { return x.free, nil }, + Wait: func(ctx context.Context, _ time.Duration) error { return ctx.Err() }, + onProgress: func(p jobs.Progress) { + x.mu.Lock() + x.progress = append(x.progress, p) + x.mu.Unlock() + }}) + return x +} + +func (x *harness) job(resumed bool) jobs.Job { + lib := int64(3) + return jobs.Job{ID: 41, Kind: Kind, LibraryID: &lib, Attempt: 1, Resumed: resumed, + Params: []byte(fmt.Sprintf(`{"librivox_id":%d,"work_id":%q,"recording_id":%q,"library_id":3}`, testID, testWork, testRec))} +} + +func (x *harness) run(ctx context.Context, resumed bool) (any, error) { + return x.h.Run(ctx, x.job(resumed), jobs.LocalReporter()) +} + +func (x *harness) path(rel string) string { return filepath.Join(x.mount, filepath.FromSlash(rel)) } + +func (x *harness) exists(rel string) bool { + _, err := os.Lstat(x.path(rel)) + return err == nil +} + +func (x *harness) steps() []string { + x.mu.Lock() + defer x.mu.Unlock() + var out []string + for _, p := range x.progress { + if len(out) == 0 || out[len(out)-1] != p.Step { + out = append(out, p.Step) + } + } + return out +} + +// failureCode is the run's coded failure, "" for none. +func failureCode(t *testing.T, out any, err error) string { + t.Helper() + var f *Failure + if !errors.As(err, &f) { + return "" + } + if r, ok := out.(*Failure); !ok || r != f { + t.Fatalf("a failure's result isn't the failure: %#v", out) + } + return f.Code +} diff --git a/internal/librivoxjob/librivoxjob.go b/internal/librivoxjob/librivoxjob.go new file mode 100644 index 0000000..18cd657 --- /dev/null +++ b/internal/librivoxjob/librivoxjob.go @@ -0,0 +1,688 @@ +// Package librivoxjob runs librivox.fetch (LIBRIVOX-PLAN.md L6): one LibriVox +// recording, read from the community metadata database's LibriVox-shaped feed, +// downloaded section by section from the Internet Archive into a staging +// folder in a books library, placed as Author/Title/NN - Section.mp3 with its +// cover.jpg and .audiosilo/librivox.json, then rescanned. +package librivoxjob + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path" + "regexp" + "strings" + "time" + + "github.com/kodestar/audiosilo-bridge/internal/fsutil" + "github.com/kodestar/audiosilo-bridge/internal/importjob" + "github.com/kodestar/audiosilo-bridge/internal/jobs" + "github.com/kodestar/audiosilo-bridge/internal/link" + "github.com/kodestar/audiosilo-bridge/internal/mapping" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/netguard" + "github.com/kodestar/audiosilo-bridge/internal/transfer" +) + +// Kind is the job kind (its family, "librivox", is the Bridge feature the +// server's console and player gate Add from LibriVox on). +const Kind = "librivox.fetch" + +// The steps a fetch reports (progress.step), in order. +const ( + StepManifest = "manifest" + StepDownload = "download" + StepCover = "cover" + StepPlace = "place" + StepRescan = "rescan" +) + +// The codes a failed fetch reports (its result {code, message}); any other +// failure has none, only its error. +const ( + CodeNotWritable = "not_writable" // not a checked writable books mapping, or no valid path there + CodeExists = "exists" // the title's folder and its fallback are both taken + CodeDiskFull = "disk_full" // the library is short of room + CodeNotFound = "not_found" // meta has no such project (or it lists no files) + CodeArchiveUnavailable = "archive_unavailable" // a file isn't on archive.org, or archive.org can't be reached + CodeMetadataOff = "metadata_off" // the Bridge has no metadata base URL +) + +// Failure is a failed fetch's result and error: a code from the list above and +// a sentence for the console. It never holds a URL. +type Failure struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func (f *Failure) Error() string { return f.Message } + +func fail(code, format string, args ...any) *Failure { + return &Failure{Code: code, Message: fmt.Sprintf(format, args...)} +} + +// Params are librivox.fetch's params, built by the server: the LibriVox +// project, the work and recording the server resolved it to in meta, and the +// books library. Unknown fields are ignored (the server may add some). +type Params struct { + LibrivoxID int `json:"librivox_id"` + WorkID string `json:"work_id"` + RecordingID string `json:"recording_id"` + LibraryID int64 `json:"library_id"` +} + +// metaID is the shape of a meta work or recording id (a slug). +var metaID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$`) + +// ParseParams reads and checks the params. +func ParseParams(raw json.RawMessage) (Params, error) { + var p Params + if err := json.Unmarshal(raw, &p); err != nil { + return Params{}, fmt.Errorf("bad params: %w", err) + } + switch { + case p.LibrivoxID <= 0: + return Params{}, errors.New("bad params: librivox_id must be a LibriVox project id") + case !metaID.MatchString(p.WorkID) || !metaID.MatchString(p.RecordingID): + return Params{}, errors.New("bad params: work_id and recording_id must be meta ids") + case p.LibraryID <= 0: + return Params{}, errors.New("bad params: library_id is required") + } + return p, nil +} + +// Result is a done fetch: the book's path as the server indexed it, the +// project, and the files placed (the sections and the cover) with their bytes. +type Result struct { + LibraryID int64 `json:"library_id"` + Path string `json:"path"` + LibrivoxID int `json:"librivox_id"` + Files int `json:"files"` + Bytes int64 `json:"bytes"` +} + +// Marker is .audiosilo/librivox.json, the book's LibriVox identity in its own +// folder (decision 3; the server doesn't read it yet). +type Marker struct { + Version int `json:"version"` + LibrivoxID int `json:"librivox_id"` + WorkID string `json:"work_id"` + RecordingID string `json:"recording_id"` + ArchiveID string `json:"archive_id"` +} + +// MarkerVersion is librivox.json's version. +const MarkerVersion = 1 + +// markerRel is the marker inside the book's folder. +const markerRel = ".audiosilo/librivox.json" + +// Meta is the metadata client a fetch reads (*metaops.Client). +type Meta interface { + LibrivoxAudiobook(ctx context.Context, id int) (metaops.LibrivoxBook, error) + LibrivoxWork(ctx context.Context, workID string) (metaops.LibrivoxWork, error) +} + +// Fetcher is the guarded HTTP client the files come through (*netguard.Client). +type Fetcher interface { + Stream(ctx context.Context, in netguard.StreamRequest) (*netguard.StreamResponse, error) + Fetch(ctx context.Context, in netguard.Request) (*netguard.Response, error) +} + +// Server is the part of the server link a fetch calls (*link.Client). +type Server interface { + importjob.BookLister + Rescan(ctx context.Context, libraryID int64, path string) (string, error) +} + +// Libraries is the mapping registry a fetch checks (*mapping.Registry). +type Libraries interface { + Library(id int64) (link.Library, bool) + Status(id int64) (link.LibraryStatus, bool) +} + +// Options wire the handler. +type Options struct { + Meta Meta + Fetcher Fetcher + Server func() Server + Libraries Libraries + Mount mapping.MountFunc + // StateDir is <data>/librivox: each fetch's plan and partial downloads' + // validators (state). + StateDir string + Log *slog.Logger + + // Seams (tests): the free-space reader and the wait between attempts and + // between looks at the library's mapping. + Free func(dir string) (int64, error) + Wait func(ctx context.Context, d time.Duration) error + // onProgress sees every progress the job reports (tests). + onProgress func(jobs.Progress) +} + +// Handler runs librivox.fetch on the download lane (capacity 1, shared with +// audible.import): one book at a time. +type Handler struct { + o Options +} + +// New returns the handler. +func New(o Options) *Handler { + if o.Log == nil { + o.Log = slog.New(slog.DiscardHandler) + } + if o.Free == nil { + o.Free = fsutil.FreeBytes + } + if o.Wait == nil { + o.Wait = netguard.Wait + } + return &Handler{o: o} +} + +// Kind is librivox.fetch. +func (h *Handler) Kind() string { return Kind } + +// Lane is the download lane. +func (h *Handler) Lane() string { return jobs.LaneDownload } + +// mappingPoll is how often a job waiting on its library's mapping looks again. +const mappingPoll = time.Second + +// Abandon removes the staging folder of a job the runner ended without +// running it (cancelled before it resumed). A lost claim keeps it: the server +// hands the job out again and the next claim resumes it. A placed book stays. +func (h *Handler) Abandon(ctx context.Context, job jobs.Job) { + if jobs.Lost(ctx) { + return + } + p, err := ParseParams(job.Params) + if err != nil { + return + } + st := h.loadState(p.LibrivoxID) + if st == nil || st.Placed { + return + } + if lib, ok := h.o.Libraries.Library(st.LibraryID); ok { + if mount, _ := h.o.Mount(lib.ID, lib.Root); mount != "" { + h.removeStaging(mount, p.LibrivoxID, st) + } + } + h.dropState(p.LibrivoxID) +} + +// Run fetches the recording: manifest, download, cover, place, rescan. +func (h *Handler) Run(ctx context.Context, job jobs.Job, rep *jobs.Reporter) (any, error) { + p, err := ParseParams(job.Params) + if err != nil { + return nil, err + } + if job.LibraryID != nil && *job.LibraryID != p.LibraryID { + return nil, errors.New("bad params: library_id isn't the job's library") + } + res, err := h.run(ctx, job, p, rep) + var f *Failure + if errors.As(err, &f) { + return f, f + } + if err != nil { + return nil, err + } + return res, nil +} + +// fetch is one run's working set. +type fetch struct { + p Params + lib link.Library + mount string + st *state + rep *jobs.Reporter +} + +func (h *Handler) run(ctx context.Context, job jobs.Job, p Params, rep *jobs.Reporter) (res *Result, err error) { + var srv Server + if h.o.Server != nil { + srv = h.o.Server() + } + if srv == nil { + return nil, errors.New("the Bridge isn't paired with a server") + } + lib, mount, err := h.mapped(ctx, p.LibraryID, rep, StepManifest, !job.Resumed) + if err != nil { + return nil, err + } + f := &fetch{p: p, lib: lib, mount: mount, rep: rep, st: h.loadState(p.LibrivoxID)} + if f.st != nil && f.st.LibraryID != lib.ID { + // An earlier fetch of the project for another library: its staging + // folder goes (best effort), and this one plans afresh. + h.Abandon(context.WithoutCancel(ctx), jobs.Job{Params: job.Params}) + f.st = nil + } + defer func() { + // A stop the next start resumes, and a lost claim the server hands out + // again, keep the staging folder and the state; a cancel or a failure + // before the book was placed removes both. A placed book stays, and its + // state with it when the rescan failed (the retry goes on from there). + if err == nil || f.st == nil || f.st.Placed { + return + } + if ctx.Err() != nil && (jobs.Resumes(ctx) || jobs.Lost(ctx)) { + return + } + h.removeStaging(f.mount, p.LibrivoxID, f.st) + h.dropState(p.LibrivoxID) + }() + + // An earlier run placed the book and then failed (the rescan): while its + // folder is still this project's, go on from the rescan. + if f.st != nil && f.st.Placed { + if h.holdsProject(mount, f.st.Dest, p.LibrivoxID) { + return h.finish(ctx, f, srv) + } + h.dropState(p.LibrivoxID) + f.st = nil + } + + h.report(rep, jobs.Progress{Step: StepManifest, Message: "Reading the recording from the metadata database"}) + book, work, err := h.manifest(ctx, p) + if err != nil { + return nil, err + } + names := sectionNames(book.Sections) + archiveID := archiveIDOf(p, book, work) + + if f.st == nil { + if err := h.plan(ctx, f, srv, book, work); err != nil { + return nil, err + } + if f.st.Placed { + return h.finish(ctx, f, srv) // the folder is already this project's + } + } + if err := h.prepare(f); err != nil { + return nil, err + } + if err := h.download(ctx, f, book.Sections, names); err != nil { + return nil, err + } + h.cover(ctx, f, book) + if err := ctx.Err(); err != nil { + return nil, err + } + if err := h.writeMarker(f, archiveID); err != nil { + return nil, err + } + if err := h.place(ctx, f, names); err != nil { + return nil, err + } + return h.finish(ctx, f, srv) +} + +// manifest reads the project's manifest and its work from meta. A work meta +// doesn't have (another meta base than the server's, decision 5) is no +// failure: the project names the folder then. +func (h *Handler) manifest(ctx context.Context, p Params) (metaops.LibrivoxBook, *metaops.LibrivoxWork, error) { + if h.o.Meta == nil { + return metaops.LibrivoxBook{}, nil, fail(CodeMetadataOff, "The Bridge has no metadata service set (metadata.base_url), so it can't read LibriVox recordings.") + } + book, err := h.o.Meta.LibrivoxAudiobook(ctx, p.LibrivoxID) + switch { + case errors.Is(err, metaops.ErrDisabled): + return book, nil, fail(CodeMetadataOff, "The Bridge has no metadata service set (metadata.base_url), so it can't read LibriVox recordings.") + case errors.Is(err, metaops.ErrNotFound): + return book, nil, fail(CodeNotFound, "The metadata database has no LibriVox recording %d.", p.LibrivoxID) + case err != nil: + return book, nil, fmt.Errorf("reading LibriVox recording %d from the metadata database: %w", p.LibrivoxID, err) + case len(book.Sections) == 0: + return book, nil, fail(CodeNotFound, "LibriVox recording %d lists no files in the metadata database.", p.LibrivoxID) + } + for i, s := range book.Sections { + if !allowedRaw(s.ListenURL) { + return book, nil, fail(CodeArchiveUnavailable, "Section %d of LibriVox recording %d isn't a file on archive.org, the only place the Bridge downloads LibriVox from.", i+1, p.LibrivoxID) + } + } + w, err := h.o.Meta.LibrivoxWork(ctx, p.WorkID) + switch { + case errors.Is(err, metaops.ErrNotFound): + return book, nil, nil + case err != nil: + return book, nil, fmt.Errorf("reading the work of LibriVox recording %d from the metadata database: %w", p.LibrivoxID, err) + } + for _, r := range w.Recordings { + if r.ID == p.RecordingID && r.LibrivoxID > 0 && r.LibrivoxID != p.LibrivoxID { + return book, nil, fail(CodeNotFound, "The metadata database's recording %s is LibriVox recording %d now, not %d: search again.", + p.RecordingID, r.LibrivoxID, p.LibrivoxID) + } + } + return book, &w, nil +} + +// archiveIDOf is the project's Internet Archive item: the recording's in the +// work, else the feed's url_iarchive. +func archiveIDOf(p Params, book metaops.LibrivoxBook, work *metaops.LibrivoxWork) string { + if work != nil { + for _, r := range work.Recordings { + if r.ID == p.RecordingID && r.ArchiveID != "" { + return r.ArchiveID + } + } + } + return book.ArchiveID() +} + +// plan picks the book's folder and records it (f.st): the title's folder under +// the author's, else once "<Title> (LibriVox <id>)", else exists. A folder +// that already holds this project (a placement whose state was lost) is the +// book: f.st comes back Placed. +func (h *Handler) plan(ctx context.Context, f *fetch, srv Server, book metaops.LibrivoxBook, work *metaops.LibrivoxWork) error { + books, err := importjob.AllBooks(ctx, srv, f.lib.ID) + if err != nil { + return fmt.Errorf("reading the library's books on the server: %w", err) + } + author, title := bookNames(f.p.LibrivoxID, book, work) + parent, dest := destination(author, title, books) + root, err := os.OpenRoot(f.mount) + if err != nil { + return fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + } + defer func() { _ = root.Close() }() + if parent != "" { + // The author folder as the library spells it (a case-insensitive disk). + if parent, err = transfer.OnDisk(transfer.RootFS(root), parent); err != nil { + return fail(CodeNotWritable, "The Bridge can't place the book under %s in the library: %v", parent, err) + } + if err := h.parentFree(root, parent, books); err != nil { + return err + } + } + st := &state{LibraryID: f.lib.ID, Parent: parent, Staging: path.Join(parent, stagingName(f.p.LibrivoxID))} + for i, name := range []string{path.Base(dest), fallbackName(path.Base(dest), f.p.LibrivoxID)} { + at := path.Join(parent, name) + if _, err := transfer.CleanRel(at); err != nil { + return fail(CodeNotWritable, "The book's folder %q isn't a valid path in the library.", at) + } + _, err := root.Lstat(at) + switch { + case errors.Is(err, fs.ErrNotExist): + st.Dest = at + case err != nil: + return fail(CodeNotWritable, "The Bridge can't look at %s in the library: %v", at, err) + case holdsProjectIn(root, at, f.p.LibrivoxID): + st.Dest, st.Placed = at, true + st.Count, st.Bytes = countFiles(root, at) + case i == 1: + return fail(CodeExists, "%s and %s are both taken in the library: nothing was overwritten. Move or rename one of them, then add the recording again.", + path.Join(parent, path.Base(dest)), at) + default: + continue + } + break + } + if !st.Placed { + if _, err := root.Lstat(parent); parent != "" && errors.Is(err, fs.ErrNotExist) { + st.MadeParent = true + } + } + if err := h.saveState(f.p.LibrivoxID, st); err != nil { + return fmt.Errorf("recording the fetch's plan in the Bridge's data: %w", err) + } + f.st = st + return nil +} + +// parentFree refuses an author folder that is itself a book (one the server +// lists, or one holding audio): the book's folder in it would join that book. +func (h *Handler) parentFree(root *os.Root, parent string, books []link.LibraryBook) error { + for _, b := range books { + if b.Path == parent { + return fail(CodeExists, "%s is already the library's book %q, so the recording can't go in a folder inside it.", parent, b.Title) + } + } + entries, err := fs.ReadDir(transfer.RootFS(root), parent) + switch { + case errors.Is(err, fs.ErrNotExist): + return nil + case err != nil: + return fail(CodeNotWritable, "The Bridge can't read the folder %s in the library: %v", parent, err) + } + for _, e := range entries { + if !e.IsDir() && audioExts[strings.ToLower(path.Ext(e.Name()))] { + return fail(CodeExists, "%s already holds audio (%s), a book of its own, so the recording can't go in a folder inside it.", parent, e.Name()) + } + } + return nil +} + +// prepare makes the author folder and the staging folder (0755, through the +// mount's root). +func (h *Handler) prepare(f *fetch) error { + root, err := os.OpenRoot(f.mount) + if err != nil { + return fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + } + defer func() { _ = root.Close() }() + if f.st.Parent != "" { + if err := root.MkdirAll(f.st.Parent, 0o755); err != nil { + return writeFailure(err, f.st.Parent) + } + } + if err := root.Mkdir(f.st.Staging, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return writeFailure(err, f.st.Staging) + } + if st, err := root.Lstat(f.st.Staging); err != nil || !st.IsDir() { + return fail(CodeNotWritable, "The Bridge can't use %s in the library as its staging folder.", f.st.Staging) + } + return nil +} + +// writeFailure is a failed write in the library as a fetch's failure: out of +// space is disk_full, a read-only filesystem or a refusal not_writable. +func writeFailure(err error, rel string) error { + switch { + case fsutil.IsNoSpace(err): + return fail(CodeDiskFull, "The library ran out of space writing %s.", rel) + case errors.Is(err, fs.ErrPermission), fsutil.IsReadOnly(err): + return fail(CodeNotWritable, "The Bridge can't write %s in the library: %v", rel, err) + } + return fmt.Errorf("writing %s in the library: %w", rel, err) +} + +// removeStaging removes the staging folder, and the author folder this fetch +// made once it is empty, through the mount's root (best effort). +func (h *Handler) removeStaging(mount string, id int, st *state) { + root, err := os.OpenRoot(mount) + if err != nil { + return + } + defer func() { _ = root.Close() }() + if path.Base(st.Staging) == stagingName(id) { + if err := root.RemoveAll(st.Staging); err != nil { + h.o.Log.Warn("librivox: removing a fetch's staging folder failed", "librivox_id", id, "path", st.Staging, "err", err) + } + } + if st.MadeParent && st.Parent != "" { + _ = root.Remove(st.Parent) // only when empty + } +} + +// writeMarker writes .audiosilo/librivox.json into the staging folder. +func (h *Handler) writeMarker(f *fetch, archiveID string) error { + b, err := json.Marshal(Marker{Version: MarkerVersion, LibrivoxID: f.p.LibrivoxID, WorkID: f.p.WorkID, + RecordingID: f.p.RecordingID, ArchiveID: archiveID}) + if err != nil { + return err + } + root, err := os.OpenRoot(f.mount) + if err != nil { + return fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + } + defer func() { _ = root.Close() }() + dir := path.Join(f.st.Staging, path.Dir(markerRel)) + if err := root.Mkdir(dir, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return writeFailure(err, dir) + } + if err := fsutil.WriteRootAtomic(root, dir, path.Base(markerRel), b, 0o644, nil); err != nil { + return writeFailure(err, path.Join(f.st.Staging, markerRel)) + } + return nil +} + +// holdsProject reports whether the folder rel holds this project's +// librivox.json (a book an earlier run placed). +func (h *Handler) holdsProject(mount, rel string, id int) bool { + root, err := os.OpenRoot(mount) + if err != nil { + return false + } + defer func() { _ = root.Close() }() + return holdsProjectIn(root, rel, id) +} + +func holdsProjectIn(root *os.Root, rel string, id int) bool { + b, err := root.ReadFile(path.Join(rel, markerRel)) + if err != nil { + return false + } + var m Marker + return json.Unmarshal(b, &m) == nil && m.LibrivoxID == id +} + +// countFiles is how many regular files the book's folder holds outside +// .audiosilo/ (the sections and the cover), and their bytes. +func countFiles(root *os.Root, rel string) (n int, bytes int64) { + entries, err := fs.ReadDir(transfer.RootFS(root), rel) + if err != nil { + return 0, 0 + } + for _, e := range entries { + if !e.Type().IsRegular() || strings.HasPrefix(e.Name(), ".") { + continue + } + if info, err := e.Info(); err == nil { + n++ + bytes += info.Size() + } + } + return n, bytes +} + +// place renames the staging folder to the book's folder: one no-replace +// rename, after the mapping is checked again (not writable now, the job parks +// and waits with what it downloaded). A folder that turned up at the +// destination meanwhile moves the book to the fallback name once, else exists. +func (h *Handler) place(ctx context.Context, f *fetch, names []string) error { + h.report(f.rep, jobs.Progress{Step: StepPlace, Message: "Placing the book in the library"}) + if _, _, err := h.mapped(ctx, f.lib.ID, f.rep, StepPlace, false); err != nil { + return err + } + root, err := os.OpenRoot(f.mount) + if err != nil { + return fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + } + defer func() { _ = root.Close() }() + n, bytes := countFiles(root, f.st.Staging) + if n < len(names) { + return fmt.Errorf("the staging folder holds %d files, %d expected", n, len(names)) + } + primary := f.st.Dest + fallback := path.Join(f.st.Parent, fallbackName(path.Base(primary), f.p.LibrivoxID)) + if strings.HasSuffix(primary, fmt.Sprintf(" (LibriVox %d)", f.p.LibrivoxID)) { + fallback = "" + } + for _, dest := range []string{primary, fallback} { + if dest == "" { + break + } + err = fsutil.RenameNoReplace(root, f.st.Staging, dest) + if errors.Is(err, fs.ErrExist) { + continue + } + if err != nil { + return writeFailure(err, dest) + } + f.st.Dest, f.st.Placed, f.st.Count, f.st.Bytes = dest, true, n, bytes + f.st.Files = nil + if serr := h.saveState(f.p.LibrivoxID, f.st); serr != nil { + h.o.Log.Warn("librivox: recording a placed book failed", "librivox_id", f.p.LibrivoxID, "err", serr) + } + return nil + } + return fail(CodeExists, "Something else is already at %s in the library: nothing was overwritten. Move or rename it, then add the recording again.", primary) +} + +// rescanAttempts bounds the rescan's tries (the book is placed either way). +const rescanAttempts = 3 + +// finish asks the server to index the placed book and makes the result; the +// state goes once it is made. +func (h *Handler) finish(ctx context.Context, f *fetch, srv Server) (*Result, error) { + h.report(f.rep, jobs.Progress{Step: StepRescan, Message: "Asking the server to index the book"}) + var bookPath string + var err error + for attempt := 1; attempt <= rescanAttempts; attempt++ { + if bookPath, err = srv.Rescan(ctx, f.lib.ID, f.st.Dest); err == nil { + break + } + if ctx.Err() != nil || attempt == rescanAttempts { + return nil, fmt.Errorf("the book is in the library at %s, but the server couldn't index it: %w", f.st.Dest, err) + } + if werr := h.o.Wait(ctx, time.Duration(attempt)*5*time.Second); werr != nil { + return nil, werr + } + } + if bookPath == "" { + bookPath = f.st.Dest + } + h.o.Log.Info("librivox: added", "librivox_id", f.p.LibrivoxID, "library", f.lib.ID, "path", bookPath, + "files", f.st.Count, "bytes", f.st.Bytes) + h.dropState(f.p.LibrivoxID) + return &Result{LibraryID: f.lib.ID, Path: bookPath, LibrivoxID: f.p.LibrivoxID, Files: f.st.Count, Bytes: f.st.Bytes}, nil +} + +// report records the job's progress. +func (h *Handler) report(rep *jobs.Reporter, pr jobs.Progress) { + rep.Progress(pr) + if h.o.onProgress != nil { + h.o.onProgress(pr) + } +} + +// mapped is the fetch's books library and its mount once the library is a +// checked, writable mapping there (audiblejob's rule): a job claimed before +// the check waits for it; a mapping that isn't writable fails a new job +// (strict) and parks a resumed one, or one past its download, not_writable +// until it comes back. +func (h *Handler) mapped(ctx context.Context, id int64, rep *jobs.Reporter, step string, strict bool) (link.Library, string, error) { + for { + lib, ok := h.o.Libraries.Library(id) + if !ok || lib.LibraryKind() != "books" { + return link.Library{}, "", fail(CodeNotWritable, "The server lists no books library %d for the Bridge.", id) + } + mount, _ := h.o.Mount(lib.ID, lib.Root) + st, checked := h.o.Libraries.Status(lib.ID) + why := fmt.Sprintf("Library %s must be mapped and writable on the Bridge: check its mapping on the Bridge's Status tab.", lib.Name) + switch { + case checked && st.MountPath == mount && st.Mapped && st.Writable: + rep.Unpark() + return lib, mount, nil + case !checked || st.MountPath != mount: + h.report(rep, jobs.Progress{Step: step, Message: "Waiting for the library's mapping check"}) + case strict: + return link.Library{}, "", fail(CodeNotWritable, "%s", why) + default: + h.report(rep, jobs.Progress{Step: step, Message: "Needs attention: " + why, + Extra: map[string]any{"park": map[string]any{"code": CodeNotWritable}}}) + rep.Park(CodeNotWritable, why) + } + if err := h.o.Wait(ctx, mappingPoll); err != nil { + return link.Library{}, "", err + } + } +} diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go new file mode 100644 index 0000000..09f6efd --- /dev/null +++ b/internal/librivoxjob/librivoxjob_test.go @@ -0,0 +1,669 @@ +package librivoxjob + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path" + "slices" + "strings" + "syscall" + "testing" + "time" + + "github.com/kodestar/audiosilo-bridge/internal/jobs" + "github.com/kodestar/audiosilo-bridge/internal/link" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/netguard" +) + +const bookDir = "L. Leslie Brooke/Johnny Crow's Garden" + +var stagingDir = "L. Leslie Brooke/" + stagingName(testID) + +func TestFetchPlacesBook(t *testing.T) { + x := newHarness(t) + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + res, ok := out.(*Result) + if !ok { + t.Fatalf("result %#v", out) + } + want := Result{LibraryID: 3, Path: bookDir, LibrivoxID: testID, Files: 4, Bytes: 1000 + 2000 + 3000 + int64(len(jpeg))} + if *res != want { + t.Fatalf("result %+v, want %+v", *res, want) + } + b, _ := json.Marshal(res) + if string(b) != `{"library_id":3,"path":"L. Leslie Brooke/Johnny Crow's Garden","librivox_id":192,"files":4,"bytes":6014}` { + t.Fatalf("result JSON %s", b) + } + for i, name := range []string{"01 - Johnny Crow's Garden.mp3", "02 - Johnny Crow's Party.mp3", "03 - Johnny Crow's New Garden.mp3"} { + got, err := os.ReadFile(x.path(path.Join(bookDir, name))) + if err != nil || !bytes.Equal(got, x.fetch.files[sectionURL(i+1)].body) { + t.Fatalf("%s: %v", name, err) + } + if st, _ := os.Stat(x.path(path.Join(bookDir, name))); st.Mode().Perm() != 0o644 { + t.Errorf("%s mode %v", name, st.Mode().Perm()) + } + } + if got, _ := os.ReadFile(x.path(bookDir + "/cover.jpg")); !bytes.Equal(got, jpeg) { + t.Fatalf("cover %q", got) + } + // librivox.json exactly as decision 3 has it. + got, err := os.ReadFile(x.path(bookDir + "/.audiosilo/librivox.json")) + if err != nil { + t.Fatal(err) + } + if string(got) != `{"version":1,"librivox_id":192,"work_id":"johnny-crows-garden-brooke","recording_id":"johnny-crows-garden-brooke-librivox-192","archive_id":"johnny_crow_garden_librivox"}` { + t.Fatalf("librivox.json %s", got) + } + if x.exists(stagingDir) { + t.Fatal("the staging folder is left") + } + if entries, _ := os.ReadDir(x.stateDir); len(entries) != 0 { + t.Fatalf("state left: %v", entries) + } + if !slices.Equal(x.srv.rescanned, []string{bookDir}) { + t.Fatalf("rescanned %v", x.srv.rescanned) + } + if s := x.steps(); !slices.Equal(s, []string{StepManifest, StepDownload, StepCover, StepPlace, StepRescan}) { + t.Fatalf("steps %v", s) + } + // The download's done/total count sections. + var last jobs.Progress + for _, p := range x.progress { + if p.Step == StepDownload { + last = p + if p.Total != 3 || p.Done > 3 { + t.Fatalf("download progress %+v", p) + } + } + } + if last.Done != 3 { + t.Fatalf("last download progress %+v", last) + } + // Every request went through the allowlist. + for _, r := range x.fetch.seen() { + if !allowedRaw(r.url) { + t.Fatalf("requested %s", r.url) + } + } +} + +func TestFetchServerAnswersAnotherPath(t *testing.T) { + x := newHarness(t) + x.srv.answer = "L. Leslie Brooke/Johnny Crow's Garden (indexed)" + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Path != x.srv.answer { + t.Fatalf("%#v %v", out, err) + } +} + +func TestFetchNoCover(t *testing.T) { + for name, set := range map[string]func(x *harness){ + "none stated": func(x *harness) { x.meta.book.CoverartJPG = nil }, + "not on archive": func(x *harness) { c := "https://librivox.org/cover.jpg"; x.meta.book.CoverartJPG = &c }, + "not a JPEG": func(x *harness) { x.fetch.files[coverURL].body = []byte("<html>") }, + "archive refuses": func(x *harness) { x.fetch.files[coverURL].status = 404 }, + } { + t.Run(name, func(t *testing.T) { + x := newHarness(t) + set(x) + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if x.exists(bookDir+"/cover.jpg") || out.(*Result).Files != 3 { + t.Fatalf("a cover was placed: %#v", out) + } + for _, r := range x.fetch.seen() { + if strings.Contains(r.url, "librivox.org") { + t.Fatal("librivox.org was requested") + } + } + }) + } +} + +// The host allowlist: a manifest file off archive.org, or over http, fails +// before anything is requested or written; a redirect off archive.org fails +// the job and leaves nothing. +func TestFetchHostAllowlist(t *testing.T) { + for name, listen := range map[string]string{ + "librivox.org": "https://librivox.org/x.mp3", + "http": "http://www.archive.org/download/x/1.mp3", + "lookalike": "https://archive.org.example/x.mp3", + "other port": "https://archive.org:8080/x.mp3", + "not a url": "::", + "empty": "", + "user and pass": "https://u:p@archive.org/x.mp3", + } { + t.Run(name, func(t *testing.T) { + x := newHarness(t) + x.meta.book.Sections[1].ListenURL = listen + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if len(x.fetch.seen()) != 0 { + t.Fatalf("requested %v", x.fetch.seen()) + } + if entries, _ := os.ReadDir(x.mount); len(entries) != 0 { + t.Fatalf("wrote %v", entries) + } + }) + } + t.Run("redirect off archive.org", func(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].redirect = "https://librivox.org/elsewhere.mp3" + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if strings.Contains(err.Error(), "librivox.org") || strings.Contains(err.Error(), "https://") { + t.Fatalf("a URL in the message: %v", err) + } + if x.exists(stagingDir) || x.exists("L. Leslie Brooke") || x.exists(bookDir) { + t.Fatal("left files behind") + } + }) + t.Run("redirect within archive.org", func(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].redirect = "https://ia800300.us.archive.org/1/items/x/2.mp3" + if _, err := x.run(context.Background(), false); err != nil { + t.Fatal(err) + } + }) +} + +// A stop during a section keeps the staging folder: the next run skips the +// finished sections and resumes the partial one with Range/If-Range. +func TestFetchResumesPartial(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) // a stop: Resumes + x.fetch.files[sectionURL(2)].cut = []int{700} + x.fetch.files[sectionURL(2)].onCut = stop + if _, err := x.run(ctx, false); !errors.Is(err, context.Canceled) { + t.Fatalf("first run: %v", err) + } + if !x.exists(stagingDir + "/01 - Johnny Crow's Garden.mp3") { + t.Fatal("the finished section went") + } + part, err := os.ReadFile(x.path(stagingDir + "/02 - Johnny Crow's Party.mp3.part")) + if err != nil || len(part) != 700 { + t.Fatalf("partial %d bytes, %v", len(part), err) + } + before := len(x.fetch.seen()) + + out, err := x.run(context.Background(), true) + if err != nil { + t.Fatal(err) + } + reqs := x.fetch.seen()[before:] + for _, r := range reqs { + if r.url == sectionURL(1) { + t.Fatal("a finished section was downloaded again") + } + } + if reqs[0].url != sectionURL(2) || reqs[0].offset != 700 || reqs[0].etag != `"etag-2"` { + t.Fatalf("the resume asked %+v", reqs[0]) + } + got, _ := os.ReadFile(x.path(bookDir + "/02 - Johnny Crow's Party.mp3")) + if !bytes.Equal(got, x.fetch.files[sectionURL(2)].body) { + t.Fatalf("resumed file %d bytes", len(got)) + } + if out.(*Result).Files != 4 { + t.Fatalf("%#v", out) + } +} + +// A partial without a strong ETag can't be continued: the section starts again. +func TestFetchRestartsWithoutStrongETag(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].etag = `W/"weak"` + x.fetch.files[sectionURL(2)].cut = []int{500} + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(x.path(bookDir + "/02 - Johnny Crow's Party.mp3")) + if !bytes.Equal(got, x.fetch.files[sectionURL(2)].body) || out.(*Result).Bytes != 6014 { + t.Fatalf("restarted file %d bytes", len(got)) + } +} + +// Dropped connections that add to the partial go on; three in a row that add +// nothing fail archive_unavailable and remove the staging folder. +func TestFetchDroppedConnections(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(3)].cut = []int{1000, 1000, 500} + if _, err := x.run(context.Background(), false); err != nil { + t.Fatal(err) + } + x = newHarness(t) + x.fetch.files[sectionURL(3)].cut = []int{0, 0, 0} + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) || x.exists("L. Leslie Brooke") { + t.Fatal("a failure left the staging folder") + } +} + +// A file archive.org doesn't have fails at once, and the staging folder goes +// (a terminal failure), the author folder this fetch made too. +func TestFetchMissingFileCleansUp(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(3)].status = 404 + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) || x.exists("L. Leslie Brooke") { + t.Fatal("left the staging or author folder") + } + if entries, _ := os.ReadDir(x.stateDir); len(entries) != 0 { + t.Fatalf("state left: %v", entries) + } + // An author folder that was there stays. + x = newHarness(t) + if err := os.MkdirAll(x.path("L. Leslie Brooke/Other"), 0o755); err != nil { + t.Fatal(err) + } + x.fetch.files[sectionURL(3)].status = 410 + if _, err := x.run(context.Background(), false); err == nil { + t.Fatal("no failure") + } + if x.exists(stagingDir) || !x.exists("L. Leslie Brooke/Other") { + t.Fatal("the cleanup took the wrong folders") + } +} + +// An admin's cancel removes the staging folder; a lost claim keeps it. +func TestFetchCancelAndLost(t *testing.T) { + x := newHarness(t) + ctx, cancel := jobs.CancelContext(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = cancel + if _, err := x.run(ctx, false); err == nil { + t.Fatal("a cancelled run succeeded") + } + if x.exists(stagingDir) || x.exists("L. Leslie Brooke") { + t.Fatal("a cancel left the staging folder") + } + if entries, _ := os.ReadDir(x.stateDir); len(entries) != 0 { + t.Fatalf("state left: %v", entries) + } + + x = newHarness(t) + ctx, lose := jobs.LoseContext(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = lose + if _, err := x.run(ctx, false); err == nil { + t.Fatal("a lost run succeeded") + } + if !x.exists(stagingDir + "/02 - Johnny Crow's Party.mp3.part") { + t.Fatal("a lost claim dropped the partial") + } +} + +func TestAbandon(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = stop + _, _ = x.run(ctx, false) + if !x.exists(stagingDir) { + t.Fatal("no staging folder to abandon") + } + x.h.Abandon(jobs.AbandonContext(context.Background(), true), x.job(true)) + if !x.exists(stagingDir) { + t.Fatal("a lost claim's Abandon removed the staging folder") + } + x.h.Abandon(jobs.AbandonContext(context.Background(), false), x.job(true)) + if x.exists(stagingDir) || x.exists("L. Leslie Brooke") { + t.Fatal("Abandon left the staging folder") + } + if entries, _ := os.ReadDir(x.stateDir); len(entries) != 0 { + t.Fatalf("state left: %v", entries) + } +} + +// The title's folder taken: once "<Title> (LibriVox <id>)", then exists with +// nothing written. +func TestFetchExistsFallback(t *testing.T) { + x := newHarness(t) + if err := os.MkdirAll(x.path(bookDir), 0o755); err != nil { + t.Fatal(err) + } + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if p := out.(*Result).Path; p != bookDir+" (LibriVox 192)" { + t.Fatalf("placed at %q", p) + } + if entries, _ := os.ReadDir(x.path(bookDir)); len(entries) != 0 { + t.Fatal("the existing folder was written to") + } + + x = newHarness(t) + for _, d := range []string{bookDir, bookDir + " (LibriVox 192)"} { + if err := os.MkdirAll(x.path(d), 0o755); err != nil { + t.Fatal(err) + } + } + out, err = x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeExists { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) || len(x.fetch.seen()) != 0 { + t.Fatal("an exists failure downloaded or staged") + } +} + +// A folder that turns up at the destination during the download: the rename +// never replaces it and takes the fallback name. +func TestFetchPlaceRace(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(3)].cut = []int{10} + x.fetch.files[sectionURL(3)].onCut = func() { _ = os.MkdirAll(x.path(bookDir+"/theirs"), 0o755) } + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if p := out.(*Result).Path; p != bookDir+" (LibriVox 192)" { + t.Fatalf("placed at %q", p) + } + if !x.exists(bookDir + "/theirs") { + t.Fatal("the other folder was replaced") + } +} + +// An author folder that is itself a book (listed, or holding audio) is never +// placed into. +func TestFetchParentIsABook(t *testing.T) { + x := newHarness(t) + x.srv.books = []link.LibraryBook{{Path: "L. Leslie Brooke", Title: "Loose", Authors: []string{"Someone Else"}}} + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeExists { + t.Fatalf("listed: code %q (%v)", code, err) + } + x = newHarness(t) + if err := os.MkdirAll(x.path("L. Leslie Brooke"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(x.path("L. Leslie Brooke/track.mp3"), []byte("a"), 0o644); err != nil { + t.Fatal(err) + } + out, err = x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeExists { + t.Fatalf("audio: code %q (%v)", code, err) + } +} + +// The author folder the library already uses is reused as it is spelled. +func TestFetchReusesAuthorFolder(t *testing.T) { + x := newHarness(t) + x.srv.books = []link.LibraryBook{{Path: "L.Leslie Brooke/Ring o' Roses", Title: "Ring o' Roses", Authors: []string{"L. Leslie Brooke"}}} + if err := os.MkdirAll(x.path("L.Leslie Brooke/Ring o' Roses"), 0o755); err != nil { + t.Fatal(err) + } + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Path != "L.Leslie Brooke/Johnny Crow's Garden" { + t.Fatalf("%#v %v", out, err) + } +} + +// A failed rescan keeps the placed book and its state: the next run goes on +// from the rescan, downloading nothing. +func TestFetchRescanFailureResumesAtRescan(t *testing.T) { + x := newHarness(t) + boom := errors.New("server busy") + x.srv.rescanErr = []error{boom, boom, boom} + if _, err := x.run(context.Background(), false); err == nil || !strings.Contains(err.Error(), "couldn't index it") { + t.Fatalf("first run: %v", err) + } + if !x.exists(bookDir + "/03 - Johnny Crow's New Garden.mp3") { + t.Fatal("a failed rescan removed the book") + } + before, reads := len(x.fetch.seen()), x.meta.reads + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if len(x.fetch.seen()) != before || x.meta.reads != reads { + t.Fatal("the retry read or downloaded again") + } + if r := out.(*Result); r.Path != bookDir || r.Files != 4 || r.Bytes != 6014 { + t.Fatalf("%+v", r) + } +} + +// A folder that already holds this project (its state lost) is the book: no +// download, only the rescan. +func TestFetchFolderAlreadyHoldsProject(t *testing.T) { + x := newHarness(t) + if err := os.MkdirAll(x.path(bookDir+"/.audiosilo"), 0o755); err != nil { + t.Fatal(err) + } + _ = os.WriteFile(x.path(bookDir+"/01 - a.mp3"), []byte("abc"), 0o644) + _ = os.WriteFile(x.path(bookDir+"/.audiosilo/librivox.json"), []byte(`{"version":1,"librivox_id":192}`), 0o644) + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if r := out.(*Result); r.Path != bookDir || r.Files != 1 || r.Bytes != 3 || len(x.fetch.seen()) != 0 { + t.Fatalf("%+v, %d requests", r, len(x.fetch.seen())) + } +} + +func TestFetchMetaFailures(t *testing.T) { + for _, tc := range []struct { + name string + set func(x *harness) + code string + }{ + {"metadata off", func(x *harness) { x.meta.bookErr = metaops.ErrDisabled }, CodeMetadataOff}, + {"no meta client", func(x *harness) { x.h.o.Meta = nil }, CodeMetadataOff}, + {"no such project", func(x *harness) { x.meta.book.ID = "1" }, CodeNotFound}, + {"no sections", func(x *harness) { x.meta.book.Sections = nil }, CodeNotFound}, + {"recording is another project", func(x *harness) { x.meta.work.Recordings[0].LibrivoxID = 5 }, CodeNotFound}, + {"meta down", func(x *harness) { x.meta.bookErr = metaops.ErrUnavailable }, ""}, + {"work unreadable", func(x *harness) { x.meta.workErr = metaops.ErrUnavailable }, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + x := newHarness(t) + tc.set(x) + out, err := x.run(context.Background(), false) + if err == nil { + t.Fatal("no failure") + } + if code := failureCode(t, out, err); code != tc.code { + t.Fatalf("code %q (%v)", code, err) + } + if entries, _ := os.ReadDir(x.mount); len(entries) != 0 { + t.Fatalf("wrote %v", entries) + } + }) + } + // A work meta doesn't have: the project names the folder. + x := newHarness(t) + x.meta.workErr = metaops.ErrNotFound + x.meta.book.Title = "Johnny Crow's Garden (version 2)" + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Path != "L. Leslie Brooke/Johnny Crow's Garden (version 2)" { + t.Fatalf("%#v %v", out, err) + } + b, _ := os.ReadFile(x.path("L. Leslie Brooke/Johnny Crow's Garden (version 2)/.audiosilo/librivox.json")) + var m Marker + if json.Unmarshal(b, &m) != nil || m.ArchiveID != testItem { + t.Fatalf("the archive id from url_iarchive: %s", b) + } +} + +func TestFetchNotWritable(t *testing.T) { + x := newHarness(t) + x.libs.writable = false + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeNotWritable { + t.Fatalf("code %q (%v)", code, err) + } + x = newHarness(t) + x.libs.kind = "podcasts" + out, err = x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeNotWritable { + t.Fatalf("podcasts: code %q (%v)", code, err) + } +} + +func TestFetchDiskFull(t *testing.T) { + x := newHarness(t) + x.free = 100 + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeDiskFull { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) { + t.Fatal("disk_full left the staging folder") + } +} + +func TestWriteFailureCodes(t *testing.T) { + var f *Failure + if !errors.As(writeFailure(syscall.ENOSPC, "x"), &f) || f.Code != CodeDiskFull { + t.Fatalf("ENOSPC: %v", f) + } + if !errors.As(writeFailure(os.ErrPermission, "x"), &f) || f.Code != CodeNotWritable { + t.Fatalf("permission: %v", f) + } + if errors.As(writeFailure(errors.New("other"), "x"), &f) { + t.Fatal("an unknown write error was coded") + } +} + +func TestParseParams(t *testing.T) { + good := `{"librivox_id":192,"work_id":"w","recording_id":"r","library_id":3,"extra":true}` + if p, err := ParseParams([]byte(good)); err != nil || p.LibrivoxID != 192 || p.LibraryID != 3 { + t.Fatalf("%+v %v", p, err) + } + for _, bad := range []string{ + `{`, `{"librivox_id":0,"work_id":"w","recording_id":"r","library_id":3}`, + `{"librivox_id":192,"work_id":"","recording_id":"r","library_id":3}`, + `{"librivox_id":192,"work_id":"../x","recording_id":"r","library_id":3}`, + `{"librivox_id":192,"work_id":"w","recording_id":"r/s","library_id":3}`, + `{"librivox_id":192,"work_id":"w","recording_id":"r","library_id":0}`, + } { + if _, err := ParseParams([]byte(bad)); err == nil { + t.Errorf("accepted %s", bad) + } + } + x := newHarness(t) + other := int64(4) + job := x.job(false) + job.LibraryID = &other + if _, err := x.h.Run(context.Background(), job, jobs.LocalReporter()); err == nil { + t.Fatal("a job for another library ran") + } +} + +// A state file that doesn't name this project's staging folder is dropped, +// never followed (it decides what the cleanup removes). +func TestStateValid(t *testing.T) { + for _, tc := range []struct { + st state + ok bool + }{ + {state{LibraryID: 3, Parent: "A", Staging: "A/" + stagingName(192), Dest: "A/T"}, true}, + {state{LibraryID: 3, Parent: "", Staging: stagingName(192), Dest: "T"}, true}, + {state{LibraryID: 3, Parent: "A", Staging: "A/" + stagingName(5), Dest: "A/T"}, false}, + {state{LibraryID: 3, Parent: "A", Staging: "B/" + stagingName(192), Dest: "A/T"}, false}, + {state{LibraryID: 3, Parent: "..", Staging: "../" + stagingName(192), Dest: "../T"}, false}, + {state{LibraryID: 3, Parent: "A", Staging: "A/" + stagingName(192), Dest: "A/../T"}, false}, + {state{LibraryID: 3, Parent: "A", Staging: "A/" + stagingName(192), Dest: "B/T"}, false}, + {state{LibraryID: 0, Parent: "A", Staging: "A/" + stagingName(192), Dest: "A/T"}, false}, + } { + if got := tc.st.valid(192); got != tc.ok { + t.Errorf("%+v valid = %v", tc.st, got) + } + } + x := newHarness(t) + _ = os.MkdirAll(x.stateDir, 0o700) + _ = os.WriteFile(x.h.statePath(192), []byte(`{"library_id":3,"parent":"..","staging":"../`+stagingName(192)+`","dest":"../T"}`), 0o600) + if x.h.loadState(192) != nil { + t.Fatal("an invalid state loaded") + } + if _, err := os.Stat(x.h.statePath(192)); err == nil { + t.Fatal("an invalid state stayed") + } +} + +// A resumed job whose library isn't writable now parks not_writable and goes +// on once the mapping is back. +func TestFetchResumedParksForMapping(t *testing.T) { + x := newHarness(t) + x.libs.writable = false + waits := 0 + x.h.o.Wait = func(context.Context, time.Duration) error { + waits++ + x.libs.writable = true + return nil + } + out, err := x.run(context.Background(), true) + if err != nil || out.(*Result).Path != bookDir { + t.Fatalf("%#v %v", out, err) + } + parked := false + for _, p := range x.progress { + if park, ok := p.Extra["park"].(map[string]any); ok && park["code"] == CodeNotWritable { + parked = true + } + } + if !parked || waits == 0 { + t.Fatal("the resumed job didn't park") + } + if x.h.Kind() != "librivox.fetch" || x.h.Lane() != jobs.LaneDownload { + t.Fatal("kind or lane") + } +} + +// archive.org's 429 and 5xx are tried again; a 206 the Bridge can't prove +// continues its partial (a 416) starts the file again. +func TestFetchTransientStatuses(t *testing.T) { + x := newHarness(t) + flaky := &flakyFetcher{fakeFetcher: x.fetch, statuses: map[string][]int{ + sectionURL(1): {429, 503}, + sectionURL(2): {416}, + }} + x.h.o.Fetcher = flaky + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Bytes != 6014 { + t.Fatalf("%#v %v", out, err) + } + x = newHarness(t) + x.h.o.Fetcher = &flakyFetcher{fakeFetcher: x.fetch, statuses: map[string][]int{sectionURL(1): {503, 503, 503}}} + out, err = x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } +} + +// flakyFetcher answers a URL's queued statuses before its file. +type flakyFetcher struct { + *fakeFetcher + statuses map[string][]int +} + +func (f *flakyFetcher) Stream(ctx context.Context, in netguard.StreamRequest) (*netguard.StreamResponse, error) { + if q := f.statuses[in.URL]; len(q) > 0 { + f.statuses[in.URL] = q[1:] + return &netguard.StreamResponse{Status: q[0], Body: io.NopCloser(strings.NewReader("")), RetryAfterSeconds: 1}, nil + } + return f.fakeFetcher.Stream(ctx, in) +} diff --git a/internal/librivoxjob/names.go b/internal/librivoxjob/names.go new file mode 100644 index 0000000..bca4eec --- /dev/null +++ b/internal/librivoxjob/names.go @@ -0,0 +1,161 @@ +package librivoxjob + +import ( + "fmt" + "net/url" + "path" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "github.com/kodestar/audiosilo-bridge/internal/importjob" + "github.com/kodestar/audiosilo-bridge/internal/link" + "github.com/kodestar/audiosilo-bridge/internal/metaops" + "github.com/kodestar/audiosilo-bridge/internal/placement" + "github.com/kodestar/audiosilo-server/pkg/match" +) + +// maxName bounds a book folder's or a section's own name (bytes, before the +// "NN - " prefix, the extension, ".part" or the " (LibriVox <id>)" suffix), so +// every name stays inside the 255 bytes a filesystem allows. +const maxName = 180 + +// variousAfter: a work with more authors than this is placed under Various. +const variousAfter = 3 + +// Various is the author folder of a work with more than variousAfter authors. +const Various = "Various" + +// segment is s as one path segment: control characters and runs of spaces as +// one space, placement's Sanitize (no "/", "\\" or NUL, ":" as " -", no +// leading or trailing dot or space), cut to maxName bytes on a rune boundary +// and trimmed again. "" when nothing is left ("..", " "). +func segment(s string) string { + s = strings.Join(strings.FieldsFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }), " ") + s = placement.Sanitize(s) + if len(s) > maxName { + cut := maxName + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + s = placement.Sanitize(s[:cut]) + } + return s +} + +// bookNames is the book's author and title as placement takes them: the title +// is the work's, or the project's own when the work has more than one LibriVox +// recording (each its own folder: "Metamorphosis (version 2)"), either when the +// other is empty, else "LibriVox <id>"; the author is the first credit, Various +// over three. work is nil when meta has no such work (the project's own then). +func bookNames(id int, book metaops.LibrivoxBook, work *metaops.LibrivoxWork) (author, title string) { + var authors []string + workTitle := "" + if work != nil { + authors, workTitle = work.Authors, work.Title + } + if len(authors) == 0 { + for _, a := range book.Authors { + if name := a.Name(); name != "" { + authors = append(authors, name) + } + } + } + switch { + case len(authors) > variousAfter: + author = Various + case len(authors) > 0: + author = authors[0] + } + first, second := workTitle, book.Title + if work != nil && work.LibrivoxCount() > 1 { + first, second = book.Title, workTitle + } + for _, t := range []string{first, second, "LibriVox " + strconv.Itoa(id)} { + if title = segment(t); title != "" { + break + } + } + return segment(author), title +} + +// destination is where the book's folder goes: placement's standalone rule (an +// existing author folder of the library reused as spelled, else the sanitized +// author, then the title), library-relative, with its parent folder ("" for +// the library's root: a book with no author). The title goes verbatim, never +// through match.CleanTitle (importjob.Planner's), which would drop the very +// "(version 2)" or "Vol. 001" that tells two LibriVox projects of one work apart. +func destination(author, title string, books []link.LibraryBook) (parent, dest string) { + folder := "" + if author != "" { + folder = importjob.AuthorFolders(books)[match.Normalize(author)] + } + sug := placement.Suggest(placement.Input{Author: author, Title: title, TitleShort: title, AuthorFolder: folder}) + dest = strings.Trim(sug.RelPath, "/") + parent = path.Dir(dest) + if parent == "." { + parent = "" + } + return parent, dest +} + +// fallbackName is the folder name tried once when the title's is taken. +func fallbackName(title string, id int) string { + return fmt.Sprintf("%s (LibriVox %d)", title, id) +} + +// audioExts are the extensions a section file keeps from its URL; anything +// else is named .mp3 (LibriVox publishes its sections as MP3). +var audioExts = map[string]bool{".mp3": true, ".ogg": true, ".m4a": true, ".m4b": true, ".flac": true, ".opus": true} + +// sectionNames are the sections' file names: "NN - <title>.<ext>", NN the +// 1-based position (never LibriVox's section_number, which starts at 0 on some +// projects), zero-padded to the count's width and at least two digits, and a +// section with no title left "Section N". +func sectionNames(sections []metaops.LibrivoxSection) []string { + width := max(2, len(strconv.Itoa(len(sections)))) + out := make([]string, len(sections)) + for i, s := range sections { + n := i + 1 + name := segment(s.Title) + if name == "" { + name = "Section " + strconv.Itoa(n) + } + out[i] = fmt.Sprintf("%0*d - %s%s", width, n, name, sectionExt(s.ListenURL)) + } + return out +} + +// sectionExt is the file's extension from its URL's path (lower case), .mp3 +// when it names no audio one. +func sectionExt(raw string) string { + if u, err := url.Parse(raw); err == nil { + if ext := strings.ToLower(path.Ext(u.Path)); audioExts[ext] { + return ext + } + } + return ".mp3" +} + +// AllowedURL reports whether a fetch may request u (the first URL and every +// redirect's, netguard checking the addresses beside it): https on the default +// port, no credentials, and archive.org or a host under it. Never librivox.org +// (LIBRIVOX-PLAN.md L6 decision 7): the Bridge reads LibriVox through meta and +// its files from the Internet Archive only. +func AllowedURL(u *url.URL) bool { + if u == nil || u.Scheme != "https" || u.User != nil || u.Opaque != "" { + return false + } + if p := u.Port(); p != "" && p != "443" { + return false + } + host := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") + return host == "archive.org" || (strings.HasSuffix(host, ".archive.org") && len(host) > len(".archive.org")) +} + +// allowedRaw is AllowedURL over a URL string. +func allowedRaw(raw string) bool { + u, err := url.Parse(strings.TrimSpace(raw)) + return err == nil && AllowedURL(u) +} diff --git a/internal/librivoxjob/names_test.go b/internal/librivoxjob/names_test.go new file mode 100644 index 0000000..01053cb --- /dev/null +++ b/internal/librivoxjob/names_test.go @@ -0,0 +1,168 @@ +package librivoxjob + +import ( + "net/url" + "strings" + "testing" + "unicode/utf8" + + "github.com/kodestar/audiosilo-bridge/internal/link" + "github.com/kodestar/audiosilo-bridge/internal/metaops" +) + +func TestSegment(t *testing.T) { + long := strings.Repeat("a", 300) + wide := strings.Repeat("é", 200) // 2 bytes each + for _, tc := range []struct{ in, want string }{ + {"Johnny Crow's Garden", "Johnny Crow's Garden"}, + {"../../etc/passwd", "-..-etc-passwd"}, + {"..", ""}, + {".hack", "hack"}, + {"...And Ladies of the Club", "And Ladies of the Club"}, + {" ", ""}, + {"", ""}, + {"Title: Subtitle", "Title - Subtitle"}, + {"a\\b", "a-b"}, + {"tab\there\nnewline\x00nul", "tab here newline nul"}, + {"two spaces", "two spaces"}, + {long, strings.Repeat("a", maxName)}, + } { + if got := segment(tc.in); got != tc.want { + t.Errorf("segment(%q) = %q, want %q", tc.in, got, tc.want) + } + } + got := segment(wide) + if len(got) > maxName || !utf8.ValidString(got) || len(got) != maxName { + t.Errorf("a multi-byte over-long name cut to %d bytes (valid %v)", len(got), utf8.ValidString(got)) + } + // Cut ending in a space or a dot is trimmed again. + if got := segment(strings.Repeat("a", maxName-1) + " b"); got != strings.Repeat("a", maxName-1) { + t.Errorf("a cut name kept its trailing space: %q", got) + } +} + +func TestBookNames(t *testing.T) { + book := metaops.LibrivoxBook{Title: "Metamorphosis (version 2)", + Authors: []metaops.LibrivoxPerson{{FirstName: "Franz", LastName: "Kafka"}}} + one := &metaops.LibrivoxWork{Title: "Metamorphosis", Authors: []string{"Franz Kafka"}, + Recordings: []metaops.LibrivoxRecording{{ID: "r1", LibrivoxID: 527}, {ID: "audible", LibrivoxID: 0}}} + two := &metaops.LibrivoxWork{Title: "Metamorphosis", Authors: []string{"Franz Kafka"}, + Recordings: []metaops.LibrivoxRecording{{ID: "r1", LibrivoxID: 527}, {ID: "r2", LibrivoxID: 5214}}} + for _, tc := range []struct { + name string + book metaops.LibrivoxBook + work *metaops.LibrivoxWork + wantAuthor, wantTit string + }{ + {"one LibriVox recording: the work's title", book, one, "Franz Kafka", "Metamorphosis"}, + {"two: the project's own title", book, two, "Franz Kafka", "Metamorphosis (version 2)"}, + {"no work: the project's", book, nil, "Franz Kafka", "Metamorphosis (version 2)"}, + {"empty work title: the project's", book, &metaops.LibrivoxWork{Authors: []string{"Franz Kafka"}}, "Franz Kafka", "Metamorphosis (version 2)"}, + {"empty everything", metaops.LibrivoxBook{}, nil, "", "LibriVox 192"}, + {"dot titles", metaops.LibrivoxBook{Title: "..."}, &metaops.LibrivoxWork{Title: ".."}, "", "LibriVox 192"}, + {"first credit", book, &metaops.LibrivoxWork{Title: "T", Authors: []string{"A", "B", "C"}}, "A", "T"}, + {"Various over three", book, &metaops.LibrivoxWork{Title: "T", Authors: []string{"A", "B", "C", "D"}}, "Various", "T"}, + {"Various from the feed", metaops.LibrivoxBook{Title: "Short Story Collection Vol. 001", Authors: []metaops.LibrivoxPerson{ + {LastName: "A"}, {LastName: "B"}, {LastName: "C"}, {LastName: "D"}}}, nil, "Various", "Short Story Collection Vol. 001"}, + {"author traversal", book, &metaops.LibrivoxWork{Title: "T", Authors: []string{"../x"}}, "-x", "T"}, + } { + t.Run(tc.name, func(t *testing.T) { + a, ti := bookNames(192, tc.book, tc.work) + if a != tc.wantAuthor || ti != tc.wantTit { + t.Fatalf("got %q / %q, want %q / %q", a, ti, tc.wantAuthor, tc.wantTit) + } + }) + } +} + +func TestDestination(t *testing.T) { + books := []link.LibraryBook{{Path: "L.Leslie Brooke/Other Book", Authors: []string{"L. Leslie Brooke"}}} + for _, tc := range []struct { + author, title, parent, dest string + }{ + {"L. Leslie Brooke", "Johnny Crow's Garden", "L.Leslie Brooke", "L.Leslie Brooke/Johnny Crow's Garden"}, + {"Franz Kafka", "Metamorphosis (version 2)", "Franz Kafka", "Franz Kafka/Metamorphosis (version 2)"}, + {"", "Anonymous Tales", "", "Anonymous Tales"}, + {"Various", "Short Story Collection Vol. 001", "Various", "Various/Short Story Collection Vol. 001"}, + } { + parent, dest := destination(tc.author, tc.title, books) + if parent != tc.parent || dest != tc.dest { + t.Errorf("destination(%q, %q) = %q, %q; want %q, %q", tc.author, tc.title, parent, dest, tc.parent, tc.dest) + } + } +} + +func TestSectionNames(t *testing.T) { + sections := []metaops.LibrivoxSection{ + {SectionNumber: "0", Title: "Introduction", ListenURL: "https://archive.org/download/x/intro_64kb.mp3"}, + {SectionNumber: "1", Title: "", ListenURL: "https://archive.org/download/x/one.ogg"}, + {SectionNumber: "2", Title: "../../escape", ListenURL: "https://archive.org/download/x/two"}, + {SectionNumber: "3", Title: ".hidden", ListenURL: "https://archive.org/download/x/three.MP3?x=1"}, + {SectionNumber: "4", Title: strings.Repeat("w", 400), ListenURL: "https://archive.org/download/x/four.exe"}, + } + got := sectionNames(sections) + want := []string{ + "01 - Introduction.mp3", + "02 - Section 2.ogg", + "03 - -..-escape.mp3", + "04 - hidden.mp3", + "05 - " + strings.Repeat("w", maxName) + ".mp3", + } + for i := range want { + if got[i] != want[i] { + t.Errorf("section %d: %q, want %q", i+1, got[i], want[i]) + } + if len(got[i]+partSuffix) > 255 { + t.Errorf("section %d's partial name is %d bytes", i+1, len(got[i]+partSuffix)) + } + } + // The width is the count's, at least two digits. + many := make([]metaops.LibrivoxSection, 120) + names := sectionNames(many) + if names[0] != "001 - Section 1.mp3" || names[119] != "120 - Section 120.mp3" { + t.Errorf("120 sections: %q .. %q", names[0], names[119]) + } + if one := sectionNames(many[:1]); one[0] != "01 - Section 1.mp3" { + t.Errorf("one section: %q", one[0]) + } +} + +func TestFallbackNameFits(t *testing.T) { + name := fallbackName(strings.Repeat("t", maxName), 99999999) + if len(name) > 255 { + t.Fatalf("the fallback name is %d bytes", len(name)) + } +} + +func TestAllowedURL(t *testing.T) { + for raw, want := range map[string]bool{ + "https://archive.org/download/x/1.mp3": true, + "https://www.archive.org/download/x/1.mp3": true, + "https://ia800300.us.archive.org/12/items/x/1.mp3": true, + "https://ARCHIVE.ORG/download/x/1.mp3": true, + "https://archive.org./download/x/1.mp3": true, + "https://archive.org:443/download/x/1.mp3": true, + "http://archive.org/download/x/1.mp3": false, + "http://www.archive.org/download/x/1.mp3": false, + "https://librivox.org/x": false, + "https://www.librivox.org/x": false, + "https://archive.org.evil.example/x": false, + "https://evilarchive.org/x": false, + "https://user:pw@archive.org/x": false, + "https://archive.org:8443/x": false, + "ftp://archive.org/x": false, + "https://207.241.224.2/x": false, + "//archive.org/x": false, + } { + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + if got := AllowedURL(u); got != want { + t.Errorf("AllowedURL(%s) = %v, want %v", raw, got, want) + } + } + if AllowedURL(nil) { + t.Error("a nil URL allowed") + } +} diff --git a/internal/librivoxjob/state.go b/internal/librivoxjob/state.go new file mode 100644 index 0000000..090fb23 --- /dev/null +++ b/internal/librivoxjob/state.go @@ -0,0 +1,112 @@ +package librivoxjob + +import ( + "encoding/json" + "errors" + "io/fs" + "os" + "path" + "path/filepath" + "strconv" + + "github.com/kodestar/audiosilo-bridge/internal/fsutil" + "github.com/kodestar/audiosilo-bridge/internal/transfer" +) + +// state is a fetch's plan and progress, kept in the Bridge's data +// (<StateDir>/<librivox_id>.json) from the moment it picks the folder: a +// resumed run (a restart, a lost claim handed out again) goes on with the same +// staging folder and destination instead of planning again against a library +// that changed meanwhile, Abandon finds the staging folder to remove, and once +// the folder is placed it is the placement marker (a failed rescan retries +// from the rescan, decision 6). +type state struct { + LibraryID int64 `json:"library_id"` + // Parent is the folder the book goes in ("" for the library's root), and + // MadeParent whether this fetch created it (removed again, when empty, with + // the staging folder). + Parent string `json:"parent"` + MadeParent bool `json:"made_parent,omitempty"` + // Staging is <parent>/.audiosilo-librivox-<id>, Dest the book's folder. + Staging string `json:"staging"` + Dest string `json:"dest"` + // Files is each section's partial download: the strong ETag and the + // size its response stated, so a resume asks only for the rest. + Files map[string]partState `json:"files,omitempty"` + // Placed: the staging folder became Dest, holding Count files of Bytes. + Placed bool `json:"placed,omitempty"` + Count int `json:"count,omitempty"` + Bytes int64 `json:"bytes,omitempty"` +} + +// partState is one section's partial download. +type partState struct { + ETag string `json:"etag,omitempty"` + Total int64 `json:"total,omitempty"` +} + +// stagingName is the staging folder's name: a dot folder the server's scan +// never indexes, beside the book's destination so placing it is one rename. +func stagingName(id int) string { return ".audiosilo-librivox-" + strconv.Itoa(id) } + +// statePath is the project's state file. +func (h *Handler) statePath(id int) string { + return filepath.Join(h.o.StateDir, strconv.Itoa(id)+".json") +} + +// loadState reads the project's state, nil when there is none. One that +// can't be read, or whose paths aren't this project's staging and a clean +// library path, goes: the run plans afresh. +func (h *Handler) loadState(id int) *state { + p := h.statePath(id) + b, err := os.ReadFile(p) //nolint:gosec // G304: the job's own state in the Bridge's data + if err != nil { + return nil + } + var st state + if json.Unmarshal(b, &st) != nil || !st.valid(id) { + _ = os.Remove(p) + return nil + } + return &st +} + +// valid reports whether the state's paths are this project's: the staging +// folder and the destination clean library paths, both directly in Parent, +// the staging folder named for the project. +func (st *state) valid(id int) bool { + if st.LibraryID <= 0 || path.Base(st.Staging) != stagingName(id) { + return false + } + parent := st.Parent + if parent == "" { + parent = "." + } else if clean, err := transfer.CleanRel(parent); err != nil || clean != parent { + return false + } + if clean, err := transfer.CleanRel(st.Dest); err != nil || clean != st.Dest || path.Dir(st.Dest) != parent { + return false + } + // The staging folder's own name is a dot name, which CleanRel refuses: + // only its folder is checked (it is Parent's). + return path.Dir(st.Staging) == parent +} + +// saveState writes the project's state. +func (h *Handler) saveState(id int, st *state) error { + if err := os.MkdirAll(h.o.StateDir, 0o700); err != nil { + return err + } + b, err := json.Marshal(st) + if err != nil { + return err + } + return fsutil.WriteFileAtomic(h.statePath(id), b, 0o600) +} + +// dropState removes the project's state. +func (h *Handler) dropState(id int) { + if err := os.Remove(h.statePath(id)); err != nil && !errors.Is(err, fs.ErrNotExist) { + h.o.Log.Warn("librivox: removing a fetch's state failed", "librivox_id", id, "err", err) + } +} diff --git a/web/src/lib/generate.test.ts b/web/src/lib/generate.test.ts index c23bf06..4b6d91c 100644 --- a/web/src/lib/generate.test.ts +++ b/web/src/lib/generate.test.ts @@ -136,6 +136,7 @@ describe('descriptions', () => { expect(kindLabel('podcast.fetch')).toBe('Fetching an episode'); expect(kindLabel('podcast.retain')).toBe('Checking retained episodes'); expect(kindLabel('audible.poll')).toBe('Checking an Audible library'); + expect(kindLabel('librivox.fetch')).toBe('Adding from LibriVox'); expect(kindLabel('sidecars.contribute')).toBe('Community share'); expect(kindLabel('meta.correct')).toBe('Corrections'); expect(kindLabel('bridge.verify')).toBe('Verify'); diff --git a/web/src/lib/generate.ts b/web/src/lib/generate.ts index d47e6b5..96691b0 100644 --- a/web/src/lib/generate.ts +++ b/web/src/lib/generate.ts @@ -57,6 +57,7 @@ const KIND_LABELS: Record<string, string> = { 'podcast.fetch': 'Fetching an episode', 'podcast.retain': 'Checking retained episodes', 'audible.poll': 'Checking an Audible library', + 'librivox.fetch': 'Adding from LibriVox', }; // kindLabel words a job kind ("" for one with no label: the kind shows as it is). From 843ea30612a065316e7b4c6d243b6aeacb2f14c7 Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 16:15:43 +0000 Subject: [PATCH 05/22] librivox.fetch: bound restarted sections and re-check the mapping per section Without a strong ETag a cut attempt restarts the file from zero, so it counted as progress and a flaky link could loop for ever; an attempt now adds only when the partial holds more than before it. The mapping is checked before each section's writes, so one that goes away mid-download parks not_writable with what was fetched instead of failing and deleting it. Part of LibriVox L6-1 (Add from LibriVox). --- internal/librivoxjob/download.go | 13 ++++++-- internal/librivoxjob/librivoxjob_test.go | 42 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go index a100051..180d654 100644 --- a/internal/librivoxjob/download.go +++ b/internal/librivoxjob/download.go @@ -49,6 +49,11 @@ func (h *Handler) download(ctx context.Context, f *fetch, sections []metaops.Lib defer func() { _ = root.Close() }() n := len(sections) for i, s := range sections { + // The mapping again before each section's writes: one that went away + // parks the job (not_writable) with what it fetched, until it is back. + if _, _, err := h.mapped(ctx, f.lib.ID, f.rep, StepDownload, false); err != nil { + return err + } h.report(f.rep, jobs.Progress{Step: StepDownload, Message: fmt.Sprintf("Downloading section %d of %d from archive.org", i+1, n), Done: i, Total: n}) if st, err := root.Lstat(path.Join(f.st.Staging, names[i])); err == nil && st.Mode().IsRegular() { @@ -63,7 +68,9 @@ func (h *Handler) download(ctx context.Context, f *fetch, sections []metaops.Lib } // retry is an attempt that may be tried again: after RetryAfter when the -// server said so, and whether it added to the partial. +// server said so, and whether the partial holds more than before it (a cut +// 200 that started the file again from zero adds nothing until it passes +// what the partial held). type retry struct { retryAfter int added bool @@ -204,13 +211,13 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r _ = root.Remove(part) return fmt.Errorf("section %d is larger than the Bridge's cap for one file", i+1) case cerr != nil: - return &retry{added: n > 0, cause: "network"} + return &retry{added: size > offset, cause: "network"} case total > 0 && size != total: if size > total { _ = root.Remove(part) delete(f.st.Files, name) } - return &retry{added: n > 0, cause: "short"} + return &retry{added: size > offset, cause: "short"} } if err := root.Rename(part, final); err != nil { return writeFailure(err, final) diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index 09f6efd..0f7b1f9 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -667,3 +667,45 @@ func (f *flakyFetcher) Stream(ctx context.Context, in netguard.StreamRequest) (* } return f.fakeFetcher.Stream(ctx, in) } + +// Without a strong ETag every attempt starts the file again: cut attempts +// that never get past what the partial held count as adding nothing, so the +// section fails instead of looping. +func TestFetchWeakETagCutsFail(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].etag = `W/"weak"` + x.fetch.files[sectionURL(2)].cut = []int{500, 500, 500, 500} + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } +} + +// A mapping that goes away during the download parks the job with what it +// fetched; it goes on once the mapping is back. +func TestFetchParksMidDownload(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = func() { x.libs.writable = false } + waits := 0 + x.h.o.Wait = func(context.Context, time.Duration) error { + // The first wait is the cut section's retry; the next is the park's. + if waits++; waits > 1 { + x.libs.writable = true + } + return nil + } + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Bytes != 6014 { + t.Fatalf("%#v %v", out, err) + } + parked := false + for _, p := range x.progress { + if park, ok := p.Extra["park"].(map[string]any); ok && park["code"] == CodeNotWritable && p.Step == StepDownload { + parked = true + } + } + if !parked { + t.Fatal("the job didn't park mid-download") + } +} From f8bc304a4af458887b24c121938a63ff78753a69 Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 16:46:18 +0000 Subject: [PATCH 06/22] netguard: ask the request's Allow about a redirect in one place Fetch and Stream each spelled the redirect's allowlist check inline, beside the shared check of the first URL. Both now call one helper, so the nil rule and the refusal have a single spelling on a security control. Part of LibriVox L6-1 (Add from LibriVox). --- internal/netguard/fetch.go | 2 +- internal/netguard/guard.go | 8 +++++++- internal/netguard/stream.go | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/netguard/fetch.go b/internal/netguard/fetch.go index e58a6ea..50e9369 100644 --- a/internal/netguard/fetch.go +++ b/internal/netguard/fetch.go @@ -99,7 +99,7 @@ func (c *Client) once(ctx context.Context, in Request) (*Response, error) { if !validURL(req.URL.String()) { return &Error{Code: "invalid_feed_url", Message: "The feed redirected to an invalid URL."} } - if in.Allow != nil && !in.Allow(req.URL) { + if !allows(in.Allow, req.URL) { return blockedHost() } stripRedirectCredentials(req, via) diff --git a/internal/netguard/guard.go b/internal/netguard/guard.go index 77e0845..84020fd 100644 --- a/internal/netguard/guard.go +++ b/internal/netguard/guard.go @@ -130,7 +130,8 @@ func (c *Client) dial(ctx context.Context, network, address string) (net.Conn, e return nil, err } -// allowedBy reports whether allow (nil: every URL) admits raw. +// allowedBy reports whether allow (nil: every URL) admits raw, a request's +// first URL. func allowedBy(allow func(*url.URL) bool, raw string) bool { if allow == nil { return true @@ -139,6 +140,11 @@ func allowedBy(allow func(*url.URL) bool, raw string) bool { return err == nil && allow(u) } +// allows reports whether allow (nil: every URL) admits u, a redirect's URL. +func allows(allow func(*url.URL) bool, u *url.URL) bool { + return allow == nil || allow(u) +} + // blockedHost is a request's failure when its Allow refused a URL, the first // or a redirect's. func blockedHost() *Error { diff --git a/internal/netguard/stream.go b/internal/netguard/stream.go index 16e362d..790a37f 100644 --- a/internal/netguard/stream.go +++ b/internal/netguard/stream.go @@ -48,7 +48,7 @@ func (c *Client) Stream(ctx context.Context, in StreamRequest) (*StreamResponse, if !validURL(req.URL.String()) { return &Error{Code: "invalid_feed_url", Message: "The episode redirected to an invalid address."} } - if in.Allow != nil && !in.Allow(req.URL) { + if !allows(in.Allow, req.URL) { return blockedHost() } stripRedirectCredentials(req, via) From 4e4f90c21bda7f4807f967aa6c048620acdd80ff Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 16:46:18 +0000 Subject: [PATCH 07/22] metaops: read works/{id} for LibriVox through the package's one mirror LibrivoxWork decoded works/{id} into a second hand-written struct, though the package keeps one mirror of that payload. The payload is now a named type read by fetchWork: workDetail caches its projection and LibrivoxWork projects the same decode uncached. The two reads share their cancel / unavailable / not-found mapping, and the fields no caller read (the work's id, a section's number and playtime) are gone. The fetch test's fake checks the first URL and a redirect with one helper. Part of LibriVox L6-1 (Add from LibriVox). --- internal/librivoxjob/fake_test.go | 24 +++++------ internal/librivoxjob/names_test.go | 10 ++--- internal/metaops/coverage.go | 64 +++++++++++++++++++----------- internal/metaops/librivox.go | 59 +++++++++++---------------- internal/metaops/librivox_test.go | 3 +- 5 files changed, 81 insertions(+), 79 deletions(-) diff --git a/internal/librivoxjob/fake_test.go b/internal/librivoxjob/fake_test.go index 97b65b0..a043261 100644 --- a/internal/librivoxjob/fake_test.go +++ b/internal/librivoxjob/fake_test.go @@ -82,21 +82,19 @@ func (f *fakeFetcher) seen() []request { } func (f *fakeFetcher) resolve(raw string, allow func(*url.URL) bool) (*remote, error) { - for _, at := range []string{raw} { + refused := func(at string) bool { u, err := url.Parse(at) - if err != nil || (allow != nil && !allow(u)) { - return nil, &netguard.Error{Code: "blocked_host", Message: "blocked"} - } + return err != nil || (allow != nil && !allow(u)) + } + if refused(raw) { + return nil, &netguard.Error{Code: "blocked_host", Message: "blocked"} } r := f.files[raw] if r == nil { return &remote{status: http.StatusNotFound}, nil } - if r.redirect != "" { - u, err := url.Parse(r.redirect) - if err != nil || (allow != nil && !allow(u)) { - return nil, &netguard.Error{Code: "blocked_host", Message: "blocked"} - } + if r.redirect != "" && refused(r.redirect) { + return nil, &netguard.Error{Code: "blocked_host", Message: "blocked"} } return r, nil } @@ -260,11 +258,11 @@ func newHarness(t *testing.T) *harness { Authors: []metaops.LibrivoxPerson{{FirstName: "L. Leslie", LastName: "Brooke"}}, URLIArchive: "https://www.archive.org/details/" + testItem, CoverartJPG: &cover, Sections: []metaops.LibrivoxSection{ - {SectionNumber: "1", Title: "Johnny Crow's Garden", ListenURL: sectionURL(1)}, - {SectionNumber: "2", Title: "Johnny Crow's Party", ListenURL: sectionURL(2)}, - {SectionNumber: "3", Title: "Johnny Crow's New Garden", ListenURL: sectionURL(3)}, + {Title: "Johnny Crow's Garden", ListenURL: sectionURL(1)}, + {Title: "Johnny Crow's Party", ListenURL: sectionURL(2)}, + {Title: "Johnny Crow's New Garden", ListenURL: sectionURL(3)}, }}, - work: metaops.LibrivoxWork{ID: testWork, Title: testTitle, Authors: []string{testAuthor}, + work: metaops.LibrivoxWork{Title: testTitle, Authors: []string{testAuthor}, Recordings: []metaops.LibrivoxRecording{{ID: testRec, LibrivoxID: testID, ArchiveID: testItem}}}, } x.fetch = &fakeFetcher{files: map[string]*remote{ diff --git a/internal/librivoxjob/names_test.go b/internal/librivoxjob/names_test.go index 01053cb..d33d2c9 100644 --- a/internal/librivoxjob/names_test.go +++ b/internal/librivoxjob/names_test.go @@ -94,11 +94,11 @@ func TestDestination(t *testing.T) { func TestSectionNames(t *testing.T) { sections := []metaops.LibrivoxSection{ - {SectionNumber: "0", Title: "Introduction", ListenURL: "https://archive.org/download/x/intro_64kb.mp3"}, - {SectionNumber: "1", Title: "", ListenURL: "https://archive.org/download/x/one.ogg"}, - {SectionNumber: "2", Title: "../../escape", ListenURL: "https://archive.org/download/x/two"}, - {SectionNumber: "3", Title: ".hidden", ListenURL: "https://archive.org/download/x/three.MP3?x=1"}, - {SectionNumber: "4", Title: strings.Repeat("w", 400), ListenURL: "https://archive.org/download/x/four.exe"}, + {Title: "Introduction", ListenURL: "https://archive.org/download/x/intro_64kb.mp3"}, + {Title: "", ListenURL: "https://archive.org/download/x/one.ogg"}, + {Title: "../../escape", ListenURL: "https://archive.org/download/x/two"}, + {Title: ".hidden", ListenURL: "https://archive.org/download/x/three.MP3?x=1"}, + {Title: strings.Repeat("w", 400), ListenURL: "https://archive.org/download/x/four.exe"}, } got := sectionNames(sections) want := []string{ diff --git a/internal/metaops/coverage.go b/internal/metaops/coverage.go index fba6baf..24be336 100644 --- a/internal/metaops/coverage.go +++ b/internal/metaops/coverage.go @@ -577,6 +577,45 @@ func (c *Client) lookup(ctx context.Context, kind, value string) (workID string, return v.workID, v.found, true } +// workPayload is the works/{id} payload, the fields this package reads: its +// ONE mirror (workDetail's cached projection and LibrivoxWork's both decode it). +type workPayload struct { + ID string `json:"id"` + Title string `json:"title"` + Authors []struct { + Name string `json:"name"` + } `json:"authors"` + Series []SeriesRef `json:"series"` + // Characters carries the names too, not just presence: SeriesGlossary needs + // them, and decoding them here keeps one cache entry per work (a sibling of + // the book being processed is usually already warm from the coverage path). + Characters []struct { + Name string `json:"name"` + Aliases []string `json:"aliases"` + } `json:"characters"` + Recaps []json.RawMessage `json:"recaps"` + Recordings []struct { + ID string `json:"id"` + Narrators []struct { + Name string `json:"name"` + } `json:"narrators"` + RuntimeMin int `json:"runtime_min"` + ASINs []RegionASIN `json:"asin"` + ISBNs []string `json:"isbn"` + // LibrivoxID and ArchiveID: the recording's LibriVox project and its + // Internet Archive item (librivox.go). + LibrivoxID int `json:"librivox_id"` + ArchiveID string `json:"archive_id"` + } `json:"recordings"` +} + +// fetchWork reads works/{id}, uncached. found=false is a clean 404; ok=false a +// transport failure. +func (c *Client) fetchWork(ctx context.Context, workID string) (res workPayload, found, ok bool) { + found, ok = c.getJSON(ctx, "/api/v1/works/"+url.PathEscape(workID), &res) + return res, found, ok +} + // workDetail fetches a work's title + sidecar presence, cached per work. Sidecar // keys are omitempty on the wire, so a present, non-empty array means the work // carries that sidecar. found=false is a clean 404; ok=false is a transport @@ -585,30 +624,7 @@ func (c *Client) workDetail(ctx context.Context, workID string) (v workVal, foun if cached, hit := c.works.get(workID); hit { return cached, true, true } - var res struct { - ID string `json:"id"` - Title string `json:"title"` - Series []SeriesRef `json:"series"` - // Characters carries the names too, not just presence: SeriesGlossary needs - // them, and decoding them here keeps ONE mirror of the works/{id} payload - // and one cache entry per work (a sibling of the book being processed is - // usually already warm from the coverage path). - Characters []struct { - Name string `json:"name"` - Aliases []string `json:"aliases"` - } `json:"characters"` - Recaps []json.RawMessage `json:"recaps"` - Recordings []struct { - ID string `json:"id"` - Narrators []struct { - Name string `json:"name"` - } `json:"narrators"` - RuntimeMin int `json:"runtime_min"` - ASINs []RegionASIN `json:"asin"` - ISBNs []string `json:"isbn"` - } `json:"recordings"` - } - f, okc := c.getJSON(ctx, "/api/v1/works/"+url.PathEscape(workID), &res) + res, f, okc := c.fetchWork(ctx, workID) if !okc { return workVal{}, false, false } diff --git a/internal/metaops/librivox.go b/internal/metaops/librivox.go index 7aedff4..63364e2 100644 --- a/internal/metaops/librivox.go +++ b/internal/metaops/librivox.go @@ -22,8 +22,8 @@ var ErrNotFound = errors.New("not found in the metadata database") var ErrUnavailable = errors.New("the metadata service is unavailable") // LibrivoxBook is one project of the LibriVox-shaped feed, the fields a fetch -// reads. LibriVox's ids, section numbers and playtimes are strings, and -// coverart_jpg is null when the project states none. +// reads. LibriVox's ids are strings, and coverart_jpg is null when the project +// states none. type LibrivoxBook struct { ID string `json:"id"` Title string `json:"title"` @@ -47,10 +47,8 @@ func (p LibrivoxPerson) Name() string { // LibrivoxSection is one section (one file) of a project. type LibrivoxSection struct { - SectionNumber string `json:"section_number"` - Title string `json:"title"` - ListenURL string `json:"listen_url"` - Playtime string `json:"playtime"` + Title string `json:"title"` + ListenURL string `json:"listen_url"` } // librivoxDetails is the prefix the feed states url_iarchive with. @@ -83,13 +81,8 @@ func (c *Client) LibrivoxAudiobook(ctx context.Context, id int) (LibrivoxBook, e Books []LibrivoxBook `json:"books"` } found, ok := c.getJSON(ctx, "/api/v1/librivox/audiobooks?"+q.Encode(), &res) - switch { - case ctx.Err() != nil: - return LibrivoxBook{}, ctx.Err() - case !ok: - return LibrivoxBook{}, ErrUnavailable - case !found: - return LibrivoxBook{}, ErrNotFound + if err := readErr(ctx, found, ok); err != nil { + return LibrivoxBook{}, err } for _, b := range res.Books { if b.ID == strconv.Itoa(id) { @@ -102,7 +95,6 @@ func (c *Client) LibrivoxAudiobook(ctx context.Context, id int) (LibrivoxBook, e // LibrivoxWork is a work as a fetch reads it: its title, its authors' names // and its recordings' LibriVox identities. type LibrivoxWork struct { - ID string Title string Authors []string Recordings []LibrivoxRecording @@ -128,34 +120,31 @@ func (w LibrivoxWork) LibrivoxCount() int { return n } +// readErr is a getJSON outcome as these reads' error: the caller's cancel, +// ErrUnavailable, ErrNotFound, or nil. +func readErr(ctx context.Context, found, ok bool) error { + switch { + case ctx.Err() != nil: + return ctx.Err() + case !ok: + return ErrUnavailable + case !found: + return ErrNotFound + } + return nil +} + // LibrivoxWork reads a work for a LibriVox fetch. ErrDisabled, ErrNotFound and // ErrUnavailable as LibrivoxAudiobook's. func (c *Client) LibrivoxWork(ctx context.Context, workID string) (LibrivoxWork, error) { if !c.Enabled() { return LibrivoxWork{}, ErrDisabled } - var res struct { - ID string `json:"id"` - Title string `json:"title"` - Authors []struct { - Name string `json:"name"` - } `json:"authors"` - Recordings []struct { - ID string `json:"id"` - LibrivoxID int `json:"librivox_id"` - ArchiveID string `json:"archive_id"` - } `json:"recordings"` - } - found, ok := c.getJSON(ctx, "/api/v1/works/"+url.PathEscape(workID), &res) - switch { - case ctx.Err() != nil: - return LibrivoxWork{}, ctx.Err() - case !ok: - return LibrivoxWork{}, ErrUnavailable - case !found: - return LibrivoxWork{}, ErrNotFound + res, found, ok := c.fetchWork(ctx, workID) + if err := readErr(ctx, found, ok); err != nil { + return LibrivoxWork{}, err } - w := LibrivoxWork{ID: res.ID, Title: res.Title} + w := LibrivoxWork{Title: res.Title} for _, a := range res.Authors { if name := strings.TrimSpace(a.Name); name != "" { w.Authors = append(w.Authors, name) diff --git a/internal/metaops/librivox_test.go b/internal/metaops/librivox_test.go index a559044..a6cad1a 100644 --- a/internal/metaops/librivox_test.go +++ b/internal/metaops/librivox_test.go @@ -54,8 +54,7 @@ func TestLibrivoxReads(t *testing.T) { t.Fatal(err) } if b.ID != "192" || b.Title != "Johnny Crow's Garden" || len(b.Sections) != 2 || b.CoverartJPG == nil || - b.Sections[1].ListenURL != "https://www.archive.org/download/johnny_crow_garden_librivox/johnnycrow_02_brooke_64kb.mp3" || - b.Sections[0].SectionNumber != "1" || b.Sections[1].Playtime != "420" { + b.Sections[1].ListenURL != "https://www.archive.org/download/johnny_crow_garden_librivox/johnnycrow_02_brooke_64kb.mp3" { t.Fatalf("%+v", b) } if b.Authors[0].Name() != "L. Leslie Brooke" || b.ArchiveID() != "johnny_crow_garden_librivox" { From 39452be1b575b01f3514da60aed7d04c2d3b0315 Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 16:46:18 +0000 Subject: [PATCH 08/22] librivox.fetch: derive the staging folder and share the cleanup and root helpers The state stored the staging folder beside the parent it is always derived from, and policed the copy on load and before removal; it is computed now. Cleaning up another library's stale fetch went through Abandon with a made-up job; both call discard. plan and place build their two candidate folders without index tests or an empty-string sentinel, plan keeps the author folder's name in its message when the disk can't be read, opening the library's root and the metadata_off failure have one spelling each, transient failures are netguard.Retryable's, and a finished section no longer costs a second state write (its file is the record). The docs name podcast.fetch as sharing the lane and pkg/match as a dependency. Part of LibriVox L6-1 (Add from LibriVox). --- CLAUDE.md | 2 +- PACKAGES.md | 8 +- internal/librivoxjob/download.go | 39 +++----- internal/librivoxjob/librivoxjob.go | 112 ++++++++++++++--------- internal/librivoxjob/librivoxjob_test.go | 22 ++--- internal/librivoxjob/state.go | 31 +++---- 6 files changed, 110 insertions(+), 104 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1af7ed1..78660ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -183,7 +183,7 @@ imports `audible`); `importjob -> {link, placement, source, audiosilo-server/pkg/match}`; `audibleacct -> {audible, link, secrets, source, store}`; `audiblejob -> {audible, audibleacct, fsutil, importjob, jobs, link, mapping, placement, source, store, transfer}` (app wires both); `librivoxjob -> {fsutil, importjob, jobs, link, -mapping, metaops, netguard, placement, transfer}`; `audiblesync -> {audible, audibleacct, jobs, link, store}`; `transfer -> {fsutil, mapping, progress}`; +mapping, metaops, netguard, placement, transfer, audiosilo-server/pkg/match}`; `audiblesync -> {audible, audibleacct, jobs, link, store}`; `transfer -> {fsutil, mapping, progress}`; `jobs -> {link, store}`; `mapping -> {fsutil, link}`; `link`, `agentproto` and `workerproto` are leaves; `workers -> {asr, fsutil, scheduler, state, store, transcript, workerproto}`; `worker -> {asr, fsutil, workerproto}` diff --git a/PACKAGES.md b/PACKAGES.md index 82af066..13d6d4f 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -2608,7 +2608,8 @@ internal/ LibriVox L6 exports Sanitize (librivoxjob names its section files with it). librivoxjob/ LibriVox L6 (LIBRIVOX-PLAN.md), librivox.fetch on the download lane - (capacity 1, shared with audible.import: one book at a time). Params + (capacity 1, shared with audible.import and an ordinary podcast.fetch: + one book at a time). Params {librivox_id, work_id, recording_id, library_id} (server-built, strict; unknown fields ignored). Manifest from the Bridge's OWN meta base (metaops.LibrivoxAudiobook: /api/v1/librivox/audiobooks?id=N& @@ -2650,7 +2651,8 @@ internal/ not_writable), ONE fsutil.RenameNoReplace staging -> book folder (a folder that raced in takes the fallback once, never replaced), then the rescan (3 tries). State <data>/librivox/<id>.json (state.go: the - library, parent and whether the fetch made it, staging, destination, + library, parent and whether the fetch made it, destination (staging is + <parent>/.audiosilo-librivox-<id>, never stored), the partials' validators, then placed + count + bytes): a resumed run follows it rather than planning again, Abandon finds the staging folder by it, and once placed it is the placement marker (a failed @@ -3625,5 +3627,5 @@ and the shared clip-floor predicate `qa.ClipStartInRange`/`qa.ClipStartFloorSec` and `audiblejob -> {audible, audibleacct, fsutil, importjob, jobs, link, mapping, placement, source, store, transfer}` (Phase 4b's import; app wires both; neither imports app or api); LibriVox L6: `librivoxjob -> {fsutil, importjob, jobs, link, -mapping, metaops, netguard, placement, transfer}` (app wires it). Handlers marshal DTOs and call into the injected packages; they +mapping, metaops, netguard, placement, transfer, audiosilo-server/pkg/match}` (app wires it). Handlers marshal DTOs and call into the injected packages; they hold no logic (state transitions live in `state`, dispatch in `scheduler`). diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go index 180d654..d2370a1 100644 --- a/internal/librivoxjob/download.go +++ b/internal/librivoxjob/download.go @@ -42,9 +42,9 @@ const partSuffix = ".part" // whole), so a resumed run skips it and goes on with the next, resuming that // one's .part with Range/If-Range when its strong ETag was recorded. func (h *Handler) download(ctx context.Context, f *fetch, sections []metaops.LibrivoxSection, names []string) error { - root, err := os.OpenRoot(f.mount) + root, err := f.openRoot() if err != nil { - return fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + return err } defer func() { _ = root.Close() }() n := len(sections) @@ -56,7 +56,7 @@ func (h *Handler) download(ctx context.Context, f *fetch, sections []metaops.Lib } h.report(f.rep, jobs.Progress{Step: StepDownload, Message: fmt.Sprintf("Downloading section %d of %d from archive.org", i+1, n), Done: i, Total: n}) - if st, err := root.Lstat(path.Join(f.st.Staging, names[i])); err == nil && st.Mode().IsRegular() { + if st, err := root.Lstat(path.Join(f.staging(), names[i])); err == nil && st.Mode().IsRegular() { continue } if err := h.section(ctx, root, f, i, s.ListenURL, names[i]); err != nil { @@ -116,7 +116,7 @@ func (h *Handler) section(ctx context.Context, root *os.Root, f *fetch, i int, r // anything unproven starts again. The file is renamed from its .part once it // holds the size the response stated. func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, rawURL, name string) error { - final := path.Join(f.st.Staging, name) + final := path.Join(f.staging(), name) part := final + partSuffix offset := int64(0) if st, err := root.Lstat(part); err == nil && st.Mode().IsRegular() { @@ -128,11 +128,8 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r } resp, err := h.o.Fetcher.Stream(ctx, netguard.StreamRequest{URL: rawURL, Offset: offset, ETag: prev.ETag, Allow: AllowedURL}) if err != nil { - if ctx.Err() != nil { - return ctx.Err() - } e := netguard.Classify(err) - if retryableCode(e.Code) { + if netguard.Retryable(err) { return &retry{retryAfter: e.RetryAfterSeconds, cause: e.Code} } return fail(CodeArchiveUnavailable, "The Bridge can't download section %d of LibriVox recording %d from archive.org (%s).", @@ -141,14 +138,14 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r defer func() { _ = resp.Body.Close() }() appending := false - total := int64(0) + start, total := int64(0), int64(0) switch { case resp.Encoding != "" && resp.Encoding != "identity": return fail(CodeArchiveUnavailable, "archive.org sent section %d of LibriVox recording %d encoded (%s).", i+1, f.p.LibrivoxID, resp.Encoding) case resp.Status == http.StatusOK: total = max(resp.ContentLength, 0) case resp.Status == http.StatusPartialContent && netguard.ResumeAction(offset, prev.Total, prev.ETag, resp) == "append": - appending, total = true, prev.Total + appending, start, total = true, offset, prev.Total case resp.Status == http.StatusPartialContent, resp.Status == http.StatusRequestedRangeNotSatisfiable: // Not a continuation the Bridge can prove: the partial goes and the // next attempt asks for the whole file. @@ -163,13 +160,9 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r if total > maxFileBytes { return fmt.Errorf("section %d is larger than the Bridge's cap for one file", i+1) } - start := int64(0) - if appending { - start = offset - } if total > 0 { need := int64(math.Ceil(transfer.DefaultHeadroom * float64(total-start))) - if free, err := h.o.Free(filepath.Join(f.mount, filepath.FromSlash(f.st.Staging))); err == nil && free < need { + if free, err := h.o.Free(filepath.Join(f.mount, filepath.FromSlash(f.staging()))); err == nil && free < need { return fail(CodeDiskFull, "The library is short of space for section %d of the recording.", i+1) } } @@ -222,20 +215,12 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r if err := root.Rename(part, final); err != nil { return writeFailure(err, final) } + // The file under its own name is the record that it is finished: its + // entry goes with the next save. delete(f.st.Files, name) - if err := h.saveState(f.p.LibrivoxID, f.st); err != nil { - h.o.Log.Warn("librivox: recording a finished section failed", "librivox_id", f.p.LibrivoxID, "err", err) - } return nil } -// retryableCode reports whether a netguard failure is worth another attempt -// (a name lookup, a timeout, a dropped connection); a blocked address or host, -// an invalid URL, a TLS failure or a redirect loop is not. -func retryableCode(code string) bool { - return code == "dns" || code == "timeout" || code == "network" -} - // writeErr remembers a write's failure, so a failed copy tells the library's // fault (disk_full, not_writable) from the connection's (try again). type writeErr struct { @@ -263,7 +248,7 @@ func (h *Handler) cover(ctx context.Context, f *fetch, book metaops.LibrivoxBook return } defer func() { _ = root.Close() }() - if _, err := root.Lstat(path.Join(f.st.Staging, coverName)); err == nil { + if _, err := root.Lstat(path.Join(f.staging(), coverName)); err == nil { return } h.report(f.rep, jobs.Progress{Step: StepCover, Message: "Downloading the cover from archive.org"}) @@ -281,7 +266,7 @@ func (h *Handler) cover(ctx context.Context, f *fetch, book metaops.LibrivoxBook log.Info("librivox: the cover isn't a JPEG; the book goes without one") return } - if err := fsutil.WriteRootAtomic(root, f.st.Staging, coverName, resp.Body, 0o644, nil); err != nil && !errors.Is(err, fs.ErrExist) { + if err := fsutil.WriteRootAtomic(root, f.staging(), coverName, resp.Body, 0o644, nil); err != nil && !errors.Is(err, fs.ErrExist) { log.Warn("librivox: writing the cover failed; the book goes without one", "err", err) } } diff --git a/internal/librivoxjob/librivoxjob.go b/internal/librivoxjob/librivoxjob.go index 18cd657..92285ae 100644 --- a/internal/librivoxjob/librivoxjob.go +++ b/internal/librivoxjob/librivoxjob.go @@ -166,7 +166,7 @@ type Options struct { } // Handler runs librivox.fetch on the download lane (capacity 1, shared with -// audible.import): one book at a time. +// audible.import and an ordinary podcast.fetch): one book at a time. type Handler struct { o Options } @@ -209,12 +209,18 @@ func (h *Handler) Abandon(ctx context.Context, job jobs.Job) { if st == nil || st.Placed { return } + h.discard(p.LibrivoxID, st) +} + +// discard removes a fetch's staging folder from the library its state names +// (best effort) and the state. +func (h *Handler) discard(id int, st *state) { if lib, ok := h.o.Libraries.Library(st.LibraryID); ok { if mount, _ := h.o.Mount(lib.ID, lib.Root); mount != "" { - h.removeStaging(mount, p.LibrivoxID, st) + h.removeStaging(mount, id, st) } } - h.dropState(p.LibrivoxID) + h.dropState(id) } // Run fetches the recording: manifest, download, cover, place, rescan. @@ -246,6 +252,18 @@ type fetch struct { rep *jobs.Reporter } +// staging is the fetch's staging folder (library-relative). +func (f *fetch) staging() string { return f.st.staging(f.p.LibrivoxID) } + +// openRoot opens the library at its mount for the fetch's writes. +func (f *fetch) openRoot() (*os.Root, error) { + root, err := os.OpenRoot(f.mount) + if err != nil { + return nil, fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + } + return root, nil +} + func (h *Handler) run(ctx context.Context, job jobs.Job, p Params, rep *jobs.Reporter) (res *Result, err error) { var srv Server if h.o.Server != nil { @@ -261,8 +279,11 @@ func (h *Handler) run(ctx context.Context, job jobs.Job, p Params, rep *jobs.Rep f := &fetch{p: p, lib: lib, mount: mount, rep: rep, st: h.loadState(p.LibrivoxID)} if f.st != nil && f.st.LibraryID != lib.ID { // An earlier fetch of the project for another library: its staging - // folder goes (best effort), and this one plans afresh. - h.Abandon(context.WithoutCancel(ctx), jobs.Job{Params: job.Params}) + // folder goes (best effort; a book it placed stays), and this one plans + // afresh. + if !f.st.Placed { + h.discard(p.LibrivoxID, f.st) + } f.st = nil } defer func() { @@ -329,13 +350,16 @@ func (h *Handler) run(ctx context.Context, job jobs.Job, p Params, rep *jobs.Rep // doesn't have (another meta base than the server's, decision 5) is no // failure: the project names the folder then. func (h *Handler) manifest(ctx context.Context, p Params) (metaops.LibrivoxBook, *metaops.LibrivoxWork, error) { + metadataOff := func() error { + return fail(CodeMetadataOff, "The Bridge has no metadata service set (metadata.base_url), so it can't read LibriVox recordings.") + } if h.o.Meta == nil { - return metaops.LibrivoxBook{}, nil, fail(CodeMetadataOff, "The Bridge has no metadata service set (metadata.base_url), so it can't read LibriVox recordings.") + return metaops.LibrivoxBook{}, nil, metadataOff() } book, err := h.o.Meta.LibrivoxAudiobook(ctx, p.LibrivoxID) switch { case errors.Is(err, metaops.ErrDisabled): - return book, nil, fail(CodeMetadataOff, "The Bridge has no metadata service set (metadata.base_url), so it can't read LibriVox recordings.") + return book, nil, metadataOff() case errors.Is(err, metaops.ErrNotFound): return book, nil, fail(CodeNotFound, "The metadata database has no LibriVox recording %d.", p.LibrivoxID) case err != nil: @@ -388,23 +412,26 @@ func (h *Handler) plan(ctx context.Context, f *fetch, srv Server, book metaops.L } author, title := bookNames(f.p.LibrivoxID, book, work) parent, dest := destination(author, title, books) - root, err := os.OpenRoot(f.mount) + root, err := f.openRoot() if err != nil { - return fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + return err } defer func() { _ = root.Close() }() if parent != "" { // The author folder as the library spells it (a case-insensitive disk). - if parent, err = transfer.OnDisk(transfer.RootFS(root), parent); err != nil { + spelled, err := transfer.OnDisk(transfer.RootFS(root), parent) + if err != nil { return fail(CodeNotWritable, "The Bridge can't place the book under %s in the library: %v", parent, err) } + parent = spelled if err := h.parentFree(root, parent, books); err != nil { return err } } - st := &state{LibraryID: f.lib.ID, Parent: parent, Staging: path.Join(parent, stagingName(f.p.LibrivoxID))} - for i, name := range []string{path.Base(dest), fallbackName(path.Base(dest), f.p.LibrivoxID)} { - at := path.Join(parent, name) + st := &state{LibraryID: f.lib.ID, Parent: parent} + dest = path.Join(parent, path.Base(dest)) // under the author folder as spelled on disk + fallback := fallbackName(dest, f.p.LibrivoxID) + for _, at := range []string{dest, fallback} { if _, err := transfer.CleanRel(at); err != nil { return fail(CodeNotWritable, "The book's folder %q isn't a valid path in the library.", at) } @@ -417,13 +444,14 @@ func (h *Handler) plan(ctx context.Context, f *fetch, srv Server, book metaops.L case holdsProjectIn(root, at, f.p.LibrivoxID): st.Dest, st.Placed = at, true st.Count, st.Bytes = countFiles(root, at) - case i == 1: - return fail(CodeExists, "%s and %s are both taken in the library: nothing was overwritten. Move or rename one of them, then add the recording again.", - path.Join(parent, path.Base(dest)), at) - default: - continue } - break + if st.Dest != "" { + break + } + } + if st.Dest == "" { + return fail(CodeExists, "%s and %s are both taken in the library: nothing was overwritten. Move or rename one of them, then add the recording again.", + dest, fallback) } if !st.Placed { if _, err := root.Lstat(parent); parent != "" && errors.Is(err, fs.ErrNotExist) { @@ -463,9 +491,9 @@ func (h *Handler) parentFree(root *os.Root, parent string, books []link.LibraryB // prepare makes the author folder and the staging folder (0755, through the // mount's root). func (h *Handler) prepare(f *fetch) error { - root, err := os.OpenRoot(f.mount) + root, err := f.openRoot() if err != nil { - return fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + return err } defer func() { _ = root.Close() }() if f.st.Parent != "" { @@ -473,11 +501,12 @@ func (h *Handler) prepare(f *fetch) error { return writeFailure(err, f.st.Parent) } } - if err := root.Mkdir(f.st.Staging, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { - return writeFailure(err, f.st.Staging) + staging := f.staging() + if err := root.Mkdir(staging, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return writeFailure(err, staging) } - if st, err := root.Lstat(f.st.Staging); err != nil || !st.IsDir() { - return fail(CodeNotWritable, "The Bridge can't use %s in the library as its staging folder.", f.st.Staging) + if st, err := root.Lstat(staging); err != nil || !st.IsDir() { + return fail(CodeNotWritable, "The Bridge can't use %s in the library as its staging folder.", staging) } return nil } @@ -502,10 +531,8 @@ func (h *Handler) removeStaging(mount string, id int, st *state) { return } defer func() { _ = root.Close() }() - if path.Base(st.Staging) == stagingName(id) { - if err := root.RemoveAll(st.Staging); err != nil { - h.o.Log.Warn("librivox: removing a fetch's staging folder failed", "librivox_id", id, "path", st.Staging, "err", err) - } + if err := root.RemoveAll(st.staging(id)); err != nil { + h.o.Log.Warn("librivox: removing a fetch's staging folder failed", "librivox_id", id, "path", st.staging(id), "err", err) } if st.MadeParent && st.Parent != "" { _ = root.Remove(st.Parent) // only when empty @@ -519,17 +546,17 @@ func (h *Handler) writeMarker(f *fetch, archiveID string) error { if err != nil { return err } - root, err := os.OpenRoot(f.mount) + root, err := f.openRoot() if err != nil { - return fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + return err } defer func() { _ = root.Close() }() - dir := path.Join(f.st.Staging, path.Dir(markerRel)) + dir := path.Join(f.staging(), path.Dir(markerRel)) if err := root.Mkdir(dir, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { return writeFailure(err, dir) } if err := fsutil.WriteRootAtomic(root, dir, path.Base(markerRel), b, 0o644, nil); err != nil { - return writeFailure(err, path.Join(f.st.Staging, markerRel)) + return writeFailure(err, path.Join(f.staging(), markerRel)) } return nil } @@ -582,25 +609,22 @@ func (h *Handler) place(ctx context.Context, f *fetch, names []string) error { if _, _, err := h.mapped(ctx, f.lib.ID, f.rep, StepPlace, false); err != nil { return err } - root, err := os.OpenRoot(f.mount) + root, err := f.openRoot() if err != nil { - return fail(CodeNotWritable, "The Bridge can't open the library at its mount: %v", err) + return err } defer func() { _ = root.Close() }() - n, bytes := countFiles(root, f.st.Staging) + n, bytes := countFiles(root, f.staging()) if n < len(names) { return fmt.Errorf("the staging folder holds %d files, %d expected", n, len(names)) } primary := f.st.Dest - fallback := path.Join(f.st.Parent, fallbackName(path.Base(primary), f.p.LibrivoxID)) - if strings.HasSuffix(primary, fmt.Sprintf(" (LibriVox %d)", f.p.LibrivoxID)) { - fallback = "" + dests := []string{primary} + if !strings.HasSuffix(primary, fallbackName("", f.p.LibrivoxID)) { + dests = append(dests, fallbackName(primary, f.p.LibrivoxID)) // not the fallback already } - for _, dest := range []string{primary, fallback} { - if dest == "" { - break - } - err = fsutil.RenameNoReplace(root, f.st.Staging, dest) + for _, dest := range dests { + err = fsutil.RenameNoReplace(root, f.staging(), dest) if errors.Is(err, fs.ErrExist) { continue } diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index 0f7b1f9..bda1f78 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -573,29 +573,27 @@ func TestParseParams(t *testing.T) { } } -// A state file that doesn't name this project's staging folder is dropped, -// never followed (it decides what the cleanup removes). +// A state file whose paths aren't clean library paths is dropped, never +// followed (it decides what the cleanup removes). func TestStateValid(t *testing.T) { for _, tc := range []struct { st state ok bool }{ - {state{LibraryID: 3, Parent: "A", Staging: "A/" + stagingName(192), Dest: "A/T"}, true}, - {state{LibraryID: 3, Parent: "", Staging: stagingName(192), Dest: "T"}, true}, - {state{LibraryID: 3, Parent: "A", Staging: "A/" + stagingName(5), Dest: "A/T"}, false}, - {state{LibraryID: 3, Parent: "A", Staging: "B/" + stagingName(192), Dest: "A/T"}, false}, - {state{LibraryID: 3, Parent: "..", Staging: "../" + stagingName(192), Dest: "../T"}, false}, - {state{LibraryID: 3, Parent: "A", Staging: "A/" + stagingName(192), Dest: "A/../T"}, false}, - {state{LibraryID: 3, Parent: "A", Staging: "A/" + stagingName(192), Dest: "B/T"}, false}, - {state{LibraryID: 0, Parent: "A", Staging: "A/" + stagingName(192), Dest: "A/T"}, false}, + {state{LibraryID: 3, Parent: "A", Dest: "A/T"}, true}, + {state{LibraryID: 3, Parent: "", Dest: "T"}, true}, + {state{LibraryID: 3, Parent: "..", Dest: "../T"}, false}, + {state{LibraryID: 3, Parent: "A", Dest: "A/../T"}, false}, + {state{LibraryID: 3, Parent: "A", Dest: "B/T"}, false}, + {state{LibraryID: 0, Parent: "A", Dest: "A/T"}, false}, } { - if got := tc.st.valid(192); got != tc.ok { + if got := tc.st.valid(); got != tc.ok { t.Errorf("%+v valid = %v", tc.st, got) } } x := newHarness(t) _ = os.MkdirAll(x.stateDir, 0o700) - _ = os.WriteFile(x.h.statePath(192), []byte(`{"library_id":3,"parent":"..","staging":"../`+stagingName(192)+`","dest":"../T"}`), 0o600) + _ = os.WriteFile(x.h.statePath(192), []byte(`{"library_id":3,"parent":"..","dest":"../T"}`), 0o600) if x.h.loadState(192) != nil { t.Fatal("an invalid state loaded") } diff --git a/internal/librivoxjob/state.go b/internal/librivoxjob/state.go index 090fb23..563c885 100644 --- a/internal/librivoxjob/state.go +++ b/internal/librivoxjob/state.go @@ -27,9 +27,8 @@ type state struct { // the staging folder). Parent string `json:"parent"` MadeParent bool `json:"made_parent,omitempty"` - // Staging is <parent>/.audiosilo-librivox-<id>, Dest the book's folder. - Staging string `json:"staging"` - Dest string `json:"dest"` + // Dest is the book's folder, directly in Parent (as the staging folder is). + Dest string `json:"dest"` // Files is each section's partial download: the strong ETag and the // size its response stated, so a resume asks only for the rest. Files map[string]partState `json:"files,omitempty"` @@ -49,14 +48,17 @@ type partState struct { // never indexes, beside the book's destination so placing it is one rename. func stagingName(id int) string { return ".audiosilo-librivox-" + strconv.Itoa(id) } +// staging is the project's staging folder: <parent>/.audiosilo-librivox-<id>. +func (st *state) staging(id int) string { return path.Join(st.Parent, stagingName(id)) } + // statePath is the project's state file. func (h *Handler) statePath(id int) string { return filepath.Join(h.o.StateDir, strconv.Itoa(id)+".json") } // loadState reads the project's state, nil when there is none. One that -// can't be read, or whose paths aren't this project's staging and a clean -// library path, goes: the run plans afresh. +// can't be read, or whose paths aren't clean library paths, goes: the run +// plans afresh. func (h *Handler) loadState(id int) *state { p := h.statePath(id) b, err := os.ReadFile(p) //nolint:gosec // G304: the job's own state in the Bridge's data @@ -64,18 +66,17 @@ func (h *Handler) loadState(id int) *state { return nil } var st state - if json.Unmarshal(b, &st) != nil || !st.valid(id) { + if json.Unmarshal(b, &st) != nil || !st.valid() { _ = os.Remove(p) return nil } return &st } -// valid reports whether the state's paths are this project's: the staging -// folder and the destination clean library paths, both directly in Parent, -// the staging folder named for the project. -func (st *state) valid(id int) bool { - if st.LibraryID <= 0 || path.Base(st.Staging) != stagingName(id) { +// valid reports whether the state's paths are clean library paths: Parent, +// and the destination directly in it. +func (st *state) valid() bool { + if st.LibraryID <= 0 { return false } parent := st.Parent @@ -84,12 +85,8 @@ func (st *state) valid(id int) bool { } else if clean, err := transfer.CleanRel(parent); err != nil || clean != parent { return false } - if clean, err := transfer.CleanRel(st.Dest); err != nil || clean != st.Dest || path.Dir(st.Dest) != parent { - return false - } - // The staging folder's own name is a dot name, which CleanRel refuses: - // only its folder is checked (it is Parent's). - return path.Dir(st.Staging) == parent + clean, err := transfer.CleanRel(st.Dest) + return err == nil && clean == st.Dest && path.Dir(st.Dest) == parent } // saveState writes the project's state. From 43bfab3ba18271f2a6c12943f6ade75487450d6f Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 16:46:19 +0000 Subject: [PATCH 09/22] librivox.fetch: buffer a section's body before it is written A body arrives in reads of one TLS record (16 KB) at most and each became its own write on the .part file: several thousand writes per section, each a round trip on an SMB, NFS or FUSE mount. A 1 MiB buffer sits before the file now, flushed after a cut connection too so the partial still holds everything that arrived for the resume. Part of LibriVox L6-1 (Add from LibriVox). --- internal/librivoxjob/download.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go index d2370a1..f8d9880 100644 --- a/internal/librivoxjob/download.go +++ b/internal/librivoxjob/download.go @@ -1,6 +1,7 @@ package librivoxjob import ( + "bufio" "context" "errors" "fmt" @@ -37,6 +38,9 @@ const maxRetryAfter = 5 * time.Minute // partSuffix names a section's partial download in the staging folder. const partSuffix = ".part" +// copyBufBytes is how much of a section's body is gathered before a write. +const copyBufBytes = 1 << 20 + // download fetches every section not in the staging folder yet: a file there // under its own name is finished (it is renamed from its .part only once // whole), so a resumed run skips it and goes on with the next, resuming that @@ -188,8 +192,15 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r if err != nil { return writeFailure(err, part) } + // Buffered: a body arrives in reads of a TLS record at most, and each as + // its own write is a round trip on a network mount. The flush runs after a + // cut connection too, so the partial holds all that arrived. w := &writeErr{w: file} - n, cerr := io.Copy(w, io.LimitReader(resp.Body, maxFileBytes-start+1)) + buf := bufio.NewWriterSize(w, copyBufBytes) + n, cerr := io.Copy(buf, io.LimitReader(resp.Body, maxFileBytes-start+1)) + if err := buf.Flush(); err != nil && w.err == nil { + w.err = err + } if err := file.Sync(); err != nil && w.err == nil { w.err = err } From dea2cacfced946eaab38e9c8e81d150450cffc1d Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:10:33 +0000 Subject: [PATCH 10/22] librivox.fetch: count a section's attempts against the most its partial held An attempt counted as progress when it left the partial larger than it found it. Without a strong ETag a cut attempt starts the file again from zero, so cuts at 10 MB, 5 MB, 6 MB, 5 MB, 6 MB each 'added' to the one before and the section retried for ever, holding the download lane. section now keeps the most the partial has held and only an attempt that passes it resets the count. The count also started at one after an attempt that added, so two attempts that added nothing failed the section where the rule says three. Part of LibriVox L6-1 (Add from LibriVox). --- internal/librivoxjob/download.go | 44 +++++++++++++++--------- internal/librivoxjob/librivoxjob_test.go | 16 +++++++++ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go index f8d9880..eadabed 100644 --- a/internal/librivoxjob/download.go +++ b/internal/librivoxjob/download.go @@ -71,22 +71,33 @@ func (h *Handler) download(ctx context.Context, f *fetch, sections []metaops.Lib return nil } -// retry is an attempt that may be tried again: after RetryAfter when the -// server said so, and whether the partial holds more than before it (a cut -// 200 that started the file again from zero adds nothing until it passes -// what the partial held). +// retry is an attempt that may be tried again: after retryAfter when the +// server said so. size is what the partial holds after it (0 when the attempt +// wrote nothing or the partial went). type retry struct { retryAfter int - added bool + size int64 cause string } func (r *retry) Error() string { return r.cause } +// partSize is what a section's partial holds, 0 when there is none. +func partSize(root *os.Root, part string) int64 { + if st, err := root.Lstat(part); err == nil && st.Mode().IsRegular() { + return st.Size() + } + return 0 +} + // section downloads one section to <staging>/<name>, trying again while // attempts add to the partial (downloadAttempts in a row that add nothing // fail archive_unavailable). func (h *Handler) section(ctx context.Context, root *os.Root, f *fetch, i int, rawURL, name string) error { + // high is the most the partial has held. Only an attempt that takes it + // past that added to it: one that started the file again from zero adds + // nothing until it does, however its size compares with the last cut's. + high := partSize(root, path.Join(f.staging(), name)+partSuffix) failures := 0 for { err := h.attempt(ctx, root, f, i, rawURL, name) @@ -99,16 +110,19 @@ func (h *Handler) section(ctx context.Context, root *os.Root, f *fetch, i int, r case !errors.As(err, &r): return err } - if r.added { - failures = 0 + if r.size > high { + high, failures = r.size, 0 + } else { + failures++ } - if failures++; failures >= downloadAttempts { + if failures >= downloadAttempts { return fail(CodeArchiveUnavailable, "archive.org didn't deliver section %d of LibriVox recording %d (%s): try again later.", i+1, f.p.LibrivoxID, r.cause) } h.o.Log.Info("librivox: a section's download failed; trying again", "librivox_id", f.p.LibrivoxID, "section", i+1, - "attempt", failures, "cause", r.cause) - wait := max(time.Duration(failures*failures)*2*time.Second, min(time.Duration(r.retryAfter)*time.Second, maxRetryAfter)) + "failures", failures, "cause", r.cause) + backoff := max(failures, 1) + wait := max(time.Duration(backoff*backoff)*2*time.Second, min(time.Duration(r.retryAfter)*time.Second, maxRetryAfter)) if werr := h.o.Wait(ctx, wait); werr != nil { return werr } @@ -122,10 +136,7 @@ func (h *Handler) section(ctx context.Context, root *os.Root, f *fetch, i int, r func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, rawURL, name string) error { final := path.Join(f.staging(), name) part := final + partSuffix - offset := int64(0) - if st, err := root.Lstat(part); err == nil && st.Mode().IsRegular() { - offset = st.Size() - } + offset := partSize(root, part) prev := f.st.Files[name] if offset == 0 { prev = partState{} @@ -215,13 +226,14 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r _ = root.Remove(part) return fmt.Errorf("section %d is larger than the Bridge's cap for one file", i+1) case cerr != nil: - return &retry{added: size > offset, cause: "network"} + return &retry{size: size, cause: "network"} case total > 0 && size != total: if size > total { _ = root.Remove(part) delete(f.st.Files, name) + size = 0 } - return &retry{added: size > offset, cause: "short"} + return &retry{size: size, cause: "short"} } if err := root.Rename(part, final); err != nil { return writeFailure(err, final) diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index bda1f78..de7de1d 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -707,3 +707,19 @@ func TestFetchParksMidDownload(t *testing.T) { t.Fatal("the job didn't park mid-download") } } + +// Restarts from zero whose cuts only ever pass the LAST cut, never the most +// the partial has held, add nothing: the section fails instead of going on +// for as long as the cuts alternate. +func TestFetchRestartsNeverPassingTheirBestFail(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].etag = `W/"weak"` + x.fetch.files[sectionURL(2)].cut = []int{1000, 500, 600, 500, 600} + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if left := len(x.fetch.files[sectionURL(2)].cut); left != 1 { + t.Fatalf("%d cuts left: three attempts in a row that add nothing end the section", left) + } +} From e1a47f00299f17e174e4e3dd222d1bb41f9231e0 Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:10:44 +0000 Subject: [PATCH 11/22] librivox.fetch: drop the old partial before a restart's validators are saved A 200 that starts a file again recorded its new ETag and size in the state before the old .part was truncated. A stop between the two left the old bytes under the new validators: the resume asked Range with the new ETag, got a 206 the Bridge could prove, and appended the new tail to the old head, placing a corrupt section of the right size. The partial is removed first now. A strong ETag is also recorded only beside a stated size: without one no 206 can be proven, so the resume only spent an attempt on a Range it then had to throw away. Part of LibriVox L6-1 (Add from LibriVox). --- internal/librivoxjob/download.go | 12 +++++++- internal/librivoxjob/librivoxjob_test.go | 37 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go index eadabed..f32f077 100644 --- a/internal/librivoxjob/download.go +++ b/internal/librivoxjob/download.go @@ -175,6 +175,14 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r if total > maxFileBytes { return fmt.Errorf("section %d is larger than the Bridge's cap for one file", i+1) } + // A body from its start replaces the partial, which goes before the new + // validators are recorded: a stop between the two must never leave the old + // bytes under validators that would prove a later 206 continues them. + if !appending { + if err := root.Remove(part); err != nil && !errors.Is(err, fs.ErrNotExist) { + return writeFailure(err, part) + } + } if total > 0 { need := int64(math.Ceil(transfer.DefaultHeadroom * float64(total-start))) if free, err := h.o.Free(filepath.Join(f.mount, filepath.FromSlash(f.staging()))); err == nil && free < need { @@ -186,8 +194,10 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r if f.st.Files == nil { f.st.Files = map[string]partState{} } + // The ETag only beside a stated size: without one no 206 could be proven + // to continue the partial, so the next attempt asks for the whole file. ps := partState{Total: total} - if netguard.StrongETag(resp.ETag) { + if total > 0 && netguard.StrongETag(resp.ETag) { ps.ETag = resp.ETag } f.st.Files[name] = ps diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index de7de1d..8a65eea 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -723,3 +723,40 @@ func TestFetchRestartsNeverPassingTheirBestFail(t *testing.T) { t.Fatalf("%d cuts left: three attempts in a row that add nothing end the section", left) } } + +// A body from its start drops the old partial before its validators are +// recorded: a stop between the two can't leave the old bytes under the new +// ETag, which would prove a later 206 continues them. +func TestFetchRestartDropsPartialBeforeValidators(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{700} + x.fetch.files[sectionURL(2)].onCut = stop + if _, err := x.run(ctx, false); !errors.Is(err, context.Canceled) { + t.Fatalf("first run: %v", err) + } + part := stagingDir + "/02 - Johnny Crow's Party.mp3.part" + if !x.exists(part) { + t.Fatal("no partial to restart") + } + // The file changed on archive.org: the resume's If-Range gets the whole + // new file under a new ETag. + x.fetch.files[sectionURL(2)] = &remote{body: bytes.Repeat([]byte("n"), 2000), etag: `"etag-2b"`} + checked := false + x.h.o.Free = func(string) (int64, error) { + if !checked { // the restarted section's attempt, before its state is saved + checked = true + if x.exists(part) { + t.Error("the old partial was still there when the new validators were recorded") + } + } + return x.free, nil + } + if _, err := x.run(context.Background(), true); err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(x.path(bookDir + "/02 - Johnny Crow's Party.mp3")) + if !checked || !bytes.Equal(got, x.fetch.files[sectionURL(2)].body) { + t.Fatalf("restarted file %d bytes, checked %v", len(got), checked) + } +} From 09c02fe8edd74ab0698dfe05c6f6df71fdec79bc Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:10:44 +0000 Subject: [PATCH 12/22] librivox.fetch: never take an empty body as a finished section A 200 with Content-Length 0, or with no stated length and no bytes, skipped the size check: an empty NN - Section.mp3 was renamed into place and skipped on every resume. Zero bytes is a short body now, tried again and then archive_unavailable. Part of LibriVox L6-1 (Add from LibriVox). --- internal/librivoxjob/download.go | 4 +++- internal/librivoxjob/librivoxjob_test.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go index f32f077..ae6d6ca 100644 --- a/internal/librivoxjob/download.go +++ b/internal/librivoxjob/download.go @@ -237,7 +237,9 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r return fmt.Errorf("section %d is larger than the Bridge's cap for one file", i+1) case cerr != nil: return &retry{size: size, cause: "network"} - case total > 0 && size != total: + case size == 0, total > 0 && size != total: + // An empty body is never a section, whatever length the response + // stated; one longer than stated can't be trusted either. if size > total { _ = root.Remove(part) delete(f.st.Files, name) diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index 8a65eea..e4a6a07 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -708,6 +708,20 @@ func TestFetchParksMidDownload(t *testing.T) { } } +// An empty body is never a section, whatever length the response stated: it +// is tried again and then fails, rather than being placed as a finished file. +func TestFetchEmptyBodyFails(t *testing.T) { + x := newHarness(t) + x.fetch.files[sectionURL(2)].body = nil + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(bookDir) || x.exists(stagingDir) { + t.Fatal("an empty section was placed or left staged") + } +} + // Restarts from zero whose cuts only ever pass the LAST cut, never the most // the partial has held, add nothing: the section fails instead of going on // for as long as the cuts alternate. From 896d22e504b29e65395a199448207f7920790d35 Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:11:02 +0000 Subject: [PATCH 13/22] librivox.fetch: check the mapping before every write and fail when its mount moves The per-section check threw away the mount mapped returned, and download held one handle on the library opened before any park. A library remapped to another folder mid-fetch, or a share remounted while the job was parked, kept the job writing through the old handle: the book was placed where the server can't see it, or a write failed and the cleanup removed everything fetched. recheck now fails not_writable when the mapping's mount is no longer the one the staging folder is in, the library is opened per section, and the cover and the marker get the same check the sections and the rename had. Part of LibriVox L6-1 (Add from LibriVox). --- internal/librivoxjob/download.go | 29 +++++++++++++++--------- internal/librivoxjob/librivoxjob.go | 21 ++++++++++++++++- internal/librivoxjob/librivoxjob_test.go | 18 +++++++++++++++ 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go index ae6d6ca..e44452f 100644 --- a/internal/librivoxjob/download.go +++ b/internal/librivoxjob/download.go @@ -46,24 +46,16 @@ const copyBufBytes = 1 << 20 // whole), so a resumed run skips it and goes on with the next, resuming that // one's .part with Range/If-Range when its strong ETag was recorded. func (h *Handler) download(ctx context.Context, f *fetch, sections []metaops.LibrivoxSection, names []string) error { - root, err := f.openRoot() - if err != nil { - return err - } - defer func() { _ = root.Close() }() n := len(sections) for i, s := range sections { // The mapping again before each section's writes: one that went away // parks the job (not_writable) with what it fetched, until it is back. - if _, _, err := h.mapped(ctx, f.lib.ID, f.rep, StepDownload, false); err != nil { + if err := h.recheck(ctx, f, StepDownload); err != nil { return err } h.report(f.rep, jobs.Progress{Step: StepDownload, Message: fmt.Sprintf("Downloading section %d of %d from archive.org", i+1, n), Done: i, Total: n}) - if st, err := root.Lstat(path.Join(f.staging(), names[i])); err == nil && st.Mode().IsRegular() { - continue - } - if err := h.section(ctx, root, f, i, s.ListenURL, names[i]); err != nil { + if err := h.fetchSection(ctx, f, i, s.ListenURL, names[i]); err != nil { return err } } @@ -71,6 +63,21 @@ func (h *Handler) download(ctx context.Context, f *fetch, sections []metaops.Lib return nil } +// fetchSection downloads section i unless the staging folder holds it already. +// The library is opened for each section: a handle opened before a park may +// be on a mount that went away since. +func (h *Handler) fetchSection(ctx context.Context, f *fetch, i int, rawURL, name string) error { + root, err := f.openRoot() + if err != nil { + return err + } + defer func() { _ = root.Close() }() + if st, err := root.Lstat(path.Join(f.staging(), name)); err == nil && st.Mode().IsRegular() { + return nil + } + return h.section(ctx, root, f, i, rawURL, name) +} + // retry is an attempt that may be tried again: after retryAfter when the // server said so. size is what the partial holds after it (0 when the attempt // wrote nothing or the partial went). @@ -278,7 +285,7 @@ func (h *Handler) cover(ctx context.Context, f *fetch, book metaops.LibrivoxBook if book.CoverartJPG == nil || *book.CoverartJPG == "" { return } - root, err := os.OpenRoot(f.mount) + root, err := f.openRoot() if err != nil { return } diff --git a/internal/librivoxjob/librivoxjob.go b/internal/librivoxjob/librivoxjob.go index 92285ae..3b28d1b 100644 --- a/internal/librivoxjob/librivoxjob.go +++ b/internal/librivoxjob/librivoxjob.go @@ -333,6 +333,10 @@ func (h *Handler) run(ctx context.Context, job jobs.Job, p Params, rep *jobs.Rep if err := h.download(ctx, f, book.Sections, names); err != nil { return nil, err } + // The mapping again before the cover's and the marker's writes. + if err := h.recheck(ctx, f, StepCover); err != nil { + return nil, err + } h.cover(ctx, f, book) if err := ctx.Err(); err != nil { return nil, err @@ -606,7 +610,7 @@ func countFiles(root *os.Root, rel string) (n int, bytes int64) { // destination meanwhile moves the book to the fallback name once, else exists. func (h *Handler) place(ctx context.Context, f *fetch, names []string) error { h.report(f.rep, jobs.Progress{Step: StepPlace, Message: "Placing the book in the library"}) - if _, _, err := h.mapped(ctx, f.lib.ID, f.rep, StepPlace, false); err != nil { + if err := h.recheck(ctx, f, StepPlace); err != nil { return err } root, err := f.openRoot() @@ -678,6 +682,21 @@ func (h *Handler) report(rep *jobs.Reporter, pr jobs.Progress) { } } +// recheck is mapped for a fetch under way: the library must still be a +// checked, writable mapping (not writable now, the job parks until it is), +// and at the mount the staging folder is in. A mapping moved to another mount +// fails the fetch: what it staged isn't there. +func (h *Handler) recheck(ctx context.Context, f *fetch, step string) error { + _, mount, err := h.mapped(ctx, f.lib.ID, f.rep, step, false) + if err != nil { + return err + } + if mount != f.mount { + return fail(CodeNotWritable, "Library %s moved to another folder on the Bridge during the fetch: add the recording again.", f.lib.Name) + } + return nil +} + // mapped is the fetch's books library and its mount once the library is a // checked, writable mapping there (audiblejob's rule): a job claimed before // the check waits for it; a mapping that isn't writable fails a new job diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index e4a6a07..8686edc 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -774,3 +774,21 @@ func TestFetchRestartDropsPartialBeforeValidators(t *testing.T) { t.Fatalf("restarted file %d bytes, checked %v", len(got), checked) } } + +// A mapping moved to another mount during the fetch fails it (what it staged +// isn't there) and removes the staging folder from the mount it was in. +func TestFetchMountMovedFails(t *testing.T) { + x := newHarness(t) + old, moved := x.mount, t.TempDir() + x.fetch.files[sectionURL(1)].cut = []int{100} + x.fetch.files[sectionURL(1)].onCut = func() { x.mount, x.libs.mount = moved, moved } + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeNotWritable { + t.Fatalf("code %q (%v)", code, err) + } + for _, mount := range []string{old, moved} { + if entries, _ := os.ReadDir(mount); len(entries) != 0 { + t.Fatalf("%s holds %v after the failure", mount, entries) + } + } +} From 797d8098b8cdbc87759aa787f414a0f60b47e063 Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:11:17 +0000 Subject: [PATCH 14/22] librivox.fetch: place only the manifest's files place renamed whatever the staging folder held, and its guard counted every regular file there (the cover, a .part), so it proved nothing about the sections. A run stopped mid-section and resumed after meta's section titles changed placed the old '01 - Old Title.mp3' and a leftover .part beside the new files, and the server indexed the duplicates. sweep now removes what isn't this manifest's sections, the cover or .audiosilo/, and fails the placement when a section is missing. An entry is kept by its name or by being the same file as the one found under that name (a mount may list NFD for NFC), and the sections are looked for again after the removals so an incomplete book is never placed. Part of LibriVox L6-1 (Add from LibriVox). --- PACKAGES.md | 31 ++++++++--- internal/librivoxjob/librivoxjob.go | 71 +++++++++++++++++++++--- internal/librivoxjob/librivoxjob_test.go | 30 ++++++++++ internal/librivoxjob/names.go | 9 +++ 4 files changed, 126 insertions(+), 15 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 13d6d4f..0d95d60 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -2639,16 +2639,33 @@ internal/ exists. Staging <parent>/.audiosilo-librivox-<id>/ (a dot folder the scan skips), every write through the mount's os.Root (0755/0644): each section to <name>.part, its strong ETag and size recorded in - the state before the body, a resume asks Range/If-Range - (netguard.ResumeAction proves the 206; anything unproven starts the - file again), renamed to its name only at the stated size (a finished - file is skipped on resume); three attempts in a row that add nothing - fail archive_unavailable, as does a 4xx; the library's free space - 1.1x each file's rest, else disk_full. cover.jpg best effort (only a + the state before the body (the ETag only beside a stated size; a + body from its start removes the old partial BEFORE the new + validators are saved, so a stop between the two never leaves old + bytes a later 206 would be proven to continue), a resume asks + Range/If-Range (netguard.ResumeAction proves the 206; anything + unproven starts the file again), renamed to its name only at the + stated size and never empty (a finished file is skipped on resume); + three attempts in a row that add nothing fail archive_unavailable + (adding = past the MOST the partial has held, section's high-water + mark, so restarts from zero that only pass the last cut don't + count), as does a 4xx other than 408, 416 and 429 (tried again); + the library's free space 1.1x each file's rest, else disk_full. + The mapping is checked again before each section, before the cover + and marker, and before the rename (recheck): not writable parks; a + mapping moved to another mount fails not_writable (the staging + folder isn't there), and the library is opened per section, not + held across a park. cover.jpg best effort (only a sniffed JPEG from archive.org), then .audiosilo/librivox.json {version 1, librivox_id, work_id, recording_id, archive_id} (decision 3), the mapping checked again (a resumed or late job parks - not_writable), ONE fsutil.RenameNoReplace staging -> book folder (a + not_writable), the staging folder swept to this manifest's files + (sweep: a partial or a section an earlier run staged under a title + the manifest changed since is removed, a missing section fails; an + entry is the manifest's by its name or by os.SameFile with the file + found under that name, since a mount may list NFD for NFC, and the + sections are looked for again after the removals), + ONE fsutil.RenameNoReplace staging -> book folder (a folder that raced in takes the fallback once, never replaced), then the rescan (3 tries). State <data>/librivox/<id>.json (state.go: the library, parent and whether the fetch made it, destination (staging is diff --git a/internal/librivoxjob/librivoxjob.go b/internal/librivoxjob/librivoxjob.go index 3b28d1b..01e032d 100644 --- a/internal/librivoxjob/librivoxjob.go +++ b/internal/librivoxjob/librivoxjob.go @@ -15,6 +15,7 @@ import ( "os" "path" "regexp" + "slices" "strings" "time" @@ -618,16 +619,12 @@ func (h *Handler) place(ctx context.Context, f *fetch, names []string) error { return err } defer func() { _ = root.Close() }() - n, bytes := countFiles(root, f.staging()) - if n < len(names) { - return fmt.Errorf("the staging folder holds %d files, %d expected", n, len(names)) + if err := sweep(root, f.staging(), names); err != nil { + return err } + n, bytes := countFiles(root, f.staging()) primary := f.st.Dest - dests := []string{primary} - if !strings.HasSuffix(primary, fallbackName("", f.p.LibrivoxID)) { - dests = append(dests, fallbackName(primary, f.p.LibrivoxID)) // not the fallback already - } - for _, dest := range dests { + for _, dest := range candidates(primary, f.p.LibrivoxID) { err = fsutil.RenameNoReplace(root, f.staging(), dest) if errors.Is(err, fs.ErrExist) { continue @@ -645,6 +642,64 @@ func (h *Handler) place(ctx context.Context, f *fetch, names []string) error { return fail(CodeExists, "Something else is already at %s in the library: nothing was overwritten. Move or rename it, then add the recording again.", primary) } +// sweep leaves the staging folder holding this manifest's files only: its +// sections, the cover and .audiosilo/. Anything else there is an earlier +// run's (a partial, or a section fetched under a title the manifest has +// changed since) and goes, so it is never placed with the book; a section +// that isn't there fails the placement. An entry is the manifest's by its +// name, or by being the same file as the one found under that name: a mount +// may list a name in another form than it was given (NFD for NFC). The +// sections are looked for again after the removals, so a mount that tells +// neither way never has an incomplete book placed. +func sweep(root *os.Root, staging string, names []string) error { + sections := func() ([]fs.FileInfo, error) { + infos := make([]fs.FileInfo, 0, len(names)+2) + for i, name := range names { + st, err := root.Lstat(path.Join(staging, name)) + if err != nil || !st.Mode().IsRegular() { + return nil, fmt.Errorf("the staging folder doesn't hold section %d of the recording's %d", i+1, len(names)) + } + infos = append(infos, st) + } + return infos, nil + } + keep, err := sections() + if err != nil { + return err + } + named := map[string]bool{} + for _, name := range names { + named[name] = true + } + for _, name := range []string{coverName, path.Dir(markerRel)} { + named[name] = true + if st, err := root.Lstat(path.Join(staging, name)); err == nil { + keep = append(keep, st) + } + } + entries, err := fs.ReadDir(transfer.RootFS(root), staging) + if err != nil { + return fmt.Errorf("reading the staging folder: %w", err) + } + for _, e := range entries { + if named[e.Name()] { + continue + } + info, err := e.Info() + if err != nil { + continue // gone since the listing + } + if slices.ContainsFunc(keep, func(k fs.FileInfo) bool { return os.SameFile(k, info) }) { + continue + } + if err := root.RemoveAll(path.Join(staging, e.Name())); err != nil { + return writeFailure(err, path.Join(staging, e.Name())) + } + } + _, err = sections() + return err +} + // rescanAttempts bounds the rescan's tries (the book is placed either way). const rescanAttempts = 3 diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index 8686edc..26ba5bc 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -775,6 +775,36 @@ func TestFetchRestartDropsPartialBeforeValidators(t *testing.T) { } } +// What an earlier run staged under names the manifest no longer has (a +// partial, a section whose title changed) is never placed with the book. +func TestFetchPlacesOnlyTheManifestsFiles(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{700} + x.fetch.files[sectionURL(2)].onCut = stop + if _, err := x.run(ctx, false); !errors.Is(err, context.Canceled) { + t.Fatalf("first run: %v", err) + } + x.meta.book.Sections[0].Title = "The Garden" + x.meta.book.Sections[1].Title = "The Party" + out, err := x.run(context.Background(), true) + if err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(x.path(bookDir)) + if err != nil { + t.Fatal(err) + } + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + want := []string{".audiosilo", "01 - The Garden.mp3", "02 - The Party.mp3", "03 - Johnny Crow's New Garden.mp3", "cover.jpg"} + if !slices.Equal(names, want) || out.(*Result).Files != 4 || out.(*Result).Bytes != 6014 { + t.Fatalf("placed %v, %+v", names, out) + } +} + // A mapping moved to another mount during the fetch fails it (what it staged // isn't there) and removes the staging folder from the mount it was in. func TestFetchMountMovedFails(t *testing.T) { diff --git a/internal/librivoxjob/names.go b/internal/librivoxjob/names.go index bca4eec..74808c3 100644 --- a/internal/librivoxjob/names.go +++ b/internal/librivoxjob/names.go @@ -105,6 +105,15 @@ func fallbackName(title string, id int) string { return fmt.Sprintf("%s (LibriVox %d)", title, id) } +// candidates are the folders a fetch planned for dest may place the book at: +// dest, then once its fallback name, unless dest is the fallback already. +func candidates(dest string, id int) []string { + if strings.HasSuffix(dest, fallbackName("", id)) { + return []string{dest} + } + return []string{dest, fallbackName(dest, id)} +} + // audioExts are the extensions a section file keeps from its URL; anything // else is named .mp3 (LibriVox publishes its sections as MP3). var audioExts = map[string]bool{".mp3": true, ".ogg": true, ".m4a": true, ".m4b": true, ".flac": true, ".opus": true} From 55e4734f61be236ecdaee10c3a468a9e591123cd Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:11:34 +0000 Subject: [PATCH 15/22] librivox.fetch: adopt a placed book whose state write never landed The rename to the book's folder and the state write that records it are two steps, and a failed write was only logged. A kill or a full data folder between them left a state that said not placed: the next run downloaded the whole book again, found its own folder at the destination and placed a second copy as Title (LibriVox N). A run whose staging folder is gone and whose folder, or its fallback, holds this project's librivox.json is placed now, and goes on from the rescan. Part of LibriVox L6-1 (Add from LibriVox). --- internal/librivoxjob/librivoxjob.go | 29 ++++++++++++++++++++++ internal/librivoxjob/librivoxjob_test.go | 31 ++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/internal/librivoxjob/librivoxjob.go b/internal/librivoxjob/librivoxjob.go index 01e032d..c5f08d2 100644 --- a/internal/librivoxjob/librivoxjob.go +++ b/internal/librivoxjob/librivoxjob.go @@ -304,6 +304,9 @@ func (h *Handler) run(ctx context.Context, job jobs.Job, p Params, rep *jobs.Rep // An earlier run placed the book and then failed (the rescan): while its // folder is still this project's, go on from the rescan. + if f.st != nil { + h.adopt(f) + } if f.st != nil && f.st.Placed { if h.holdsProject(mount, f.st.Dest, p.LibrivoxID) { return h.finish(ctx, f, srv) @@ -700,6 +703,32 @@ func sweep(root *os.Root, staging string, names []string) error { return err } +// adopt marks a fetch placed whose staging folder is gone and whose folder +// (or its fallback) holds this project: the rename happened and the state's +// write after it didn't land (a kill between the two, a full data folder). +// Without it the resumed run would download the book again and place it a +// second time under the fallback name. +func (h *Handler) adopt(f *fetch) { + if f.st.Placed { + return + } + root, err := f.openRoot() + if err != nil { + return + } + defer func() { _ = root.Close() }() + if _, err := root.Lstat(f.staging()); !errors.Is(err, fs.ErrNotExist) { + return + } + for _, at := range candidates(f.st.Dest, f.p.LibrivoxID) { + if holdsProjectIn(root, at, f.p.LibrivoxID) { + f.st.Dest, f.st.Placed, f.st.Files = at, true, nil + f.st.Count, f.st.Bytes = countFiles(root, at) + return + } + } +} + // rescanAttempts bounds the rescan's tries (the book is placed either way). const rescanAttempts = 3 diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index 26ba5bc..16f442d 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -822,3 +822,34 @@ func TestFetchMountMovedFails(t *testing.T) { } } } + +// A placement whose state never said so (a kill between the rename and the +// state's write) is found by the book's marker: the next run rescans it and +// doesn't download the book again under the fallback name. +func TestFetchAdoptsUnrecordedPlacement(t *testing.T) { + x := newHarness(t) + boom := errors.New("server busy") + x.srv.rescanErr = []error{boom, boom, boom} + if _, err := x.run(context.Background(), false); err == nil { + t.Fatal("the first run's rescan didn't fail") + } + st := x.h.loadState(testID) + if st == nil || !st.Placed { + t.Fatalf("state %+v", st) + } + st.Placed, st.Count, st.Bytes = false, 0, 0 + if err := x.h.saveState(testID, st); err != nil { + t.Fatal(err) + } + before := len(x.fetch.seen()) + out, err := x.run(context.Background(), false) + if err != nil { + t.Fatal(err) + } + if r := out.(*Result); r.Path != bookDir || r.Files != 4 || r.Bytes != 6014 || len(x.fetch.seen()) != before { + t.Fatalf("%+v, %d new requests", r, len(x.fetch.seen())-before) + } + if x.exists(fallbackName(bookDir, testID)) || x.exists(stagingDir) { + t.Fatal("the book was fetched a second time") + } +} From 0d87984ac121a0047481db0f62e452545bde6d03 Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:11:34 +0000 Subject: [PATCH 16/22] librivox.fetch: Abandon leaves another library's fetch of the project alone The state is keyed by the LibriVox project only, and the server allows one open fetch of a project per library. Abandoning a cancelled job for library A while the state on disk was a later fetch of the same project into library B removed B's staging folder and state. Abandon now returns when the state names another library than the abandoned job's. The package notes also cover the placement adopted from its marker. Part of LibriVox L6-1 (Add from LibriVox). --- PACKAGES.md | 12 +++++++++--- internal/librivoxjob/librivoxjob.go | 6 ++++-- internal/librivoxjob/librivoxjob_test.go | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 0d95d60..bbf0866 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -2674,9 +2674,15 @@ internal/ follows it rather than planning again, Abandon finds the staging folder by it, and once placed it is the placement marker (a failed rescan keeps the book and the state; the retry goes on from the - rescan). A stop or a lost claim keeps staging and state; a cancel, a - failure or Abandon (not on a lost claim) removes the staging folder, - the author folder it made when empty, and the state. A lost claim + rescan). A placement the state never recorded (a kill between the + rename and the state's write) is adopted by the marker: staging + gone and the folder or its fallback holding this project is placed + (adopt), never fetched again. A stop or a lost claim keeps staging + and state; a cancel, a + failure or Abandon (not on a lost claim, and only a state for the + abandoned job's own library: the state is keyed by project) removes + the staging folder, the author folder it made when empty, and the + state. A lost claim never handed out again leaves its staging folder (hidden, unindexed). Steps manifest, download (done/total = sections), cover, place, rescan. Result {library_id, path (the rescan's), librivox_id, files, diff --git a/internal/librivoxjob/librivoxjob.go b/internal/librivoxjob/librivoxjob.go index c5f08d2..775658c 100644 --- a/internal/librivoxjob/librivoxjob.go +++ b/internal/librivoxjob/librivoxjob.go @@ -197,7 +197,9 @@ const mappingPoll = time.Second // Abandon removes the staging folder of a job the runner ended without // running it (cancelled before it resumed). A lost claim keeps it: the server -// hands the job out again and the next claim resumes it. A placed book stays. +// hands the job out again and the next claim resumes it. A placed book stays, +// and so does a state for another library: the state is the project's, not +// the job's, and that one is a later fetch's. func (h *Handler) Abandon(ctx context.Context, job jobs.Job) { if jobs.Lost(ctx) { return @@ -207,7 +209,7 @@ func (h *Handler) Abandon(ctx context.Context, job jobs.Job) { return } st := h.loadState(p.LibrivoxID) - if st == nil || st.Placed { + if st == nil || st.Placed || st.LibraryID != p.LibraryID { return } h.discard(p.LibrivoxID, st) diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index 16f442d..dd85b10 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -853,3 +853,19 @@ func TestFetchAdoptsUnrecordedPlacement(t *testing.T) { t.Fatal("the book was fetched a second time") } } + +// Abandon leaves a state that is another library's fetch of the project: the +// state is the project's, and the abandoned job never wrote that one. +func TestAbandonLeavesAnotherLibrarysFetch(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = stop + _, _ = x.run(ctx, false) + other := x.job(true) + other.Params = bytes.Replace(other.Params, []byte(`"library_id":3`), []byte(`"library_id":4`), 1) + x.h.Abandon(jobs.AbandonContext(context.Background(), false), other) + if !x.exists(stagingDir) || x.h.loadState(testID) == nil { + t.Fatal("Abandon removed another library's fetch") + } +} From 692c3475f40c8eaac7804ece7ae74329dc8b8eca Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:11:34 +0000 Subject: [PATCH 17/22] librivox.fetch: refuse a padded section or cover URL at the manifest allowedRaw checked the trimmed URL while the request, and the file's extension, used the string as it stood. A listen_url with a trailing space passed the manifest check and was then requested with %20 appended (a 404), after the staging folder was made; a leading space failed as an invalid URL. A URL with surrounding space is refused where the manifest is read, so the check and the request see one spelling. Part of LibriVox L6-1 (Add from LibriVox). --- PACKAGES.md | 3 ++- internal/librivoxjob/librivoxjob_test.go | 19 +++++++++++++++++++ internal/librivoxjob/names.go | 8 ++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index bbf0866..47cbe81 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -2619,7 +2619,8 @@ internal/ section URL must be https on archive.org or *.archive.org (AllowedURL, checked before anything is fetched or written, and on every redirect through netguard's per-request Allow; never - librivox.org), fetched by the app's own netguard client (public + librivox.org; a padded URL is refused as it stands, so the check + and the request see one spelling), fetched by the app's own netguard client (public addresses only, not the podcasts' allow_networks). Placement: placement.Suggest standalone (an existing author folder from importjob.AuthorFolders reused verbatim, else the sanitized author), diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index dd85b10..c6895f6 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -869,3 +869,22 @@ func TestAbandonLeavesAnotherLibrarysFetch(t *testing.T) { t.Fatal("Abandon removed another library's fetch") } } + +// A padded URL is refused as it stands: the check and the request see one +// spelling. +func TestFetchPaddedURLRefused(t *testing.T) { + for _, raw := range []string{" " + sectionURL(1), sectionURL(1) + " ", sectionURL(1) + "\n"} { + if allowedRaw(raw) { + t.Errorf("allowedRaw(%q)", raw) + } + } + if !allowedRaw(sectionURL(1)) { + t.Error("the URL itself refused") + } + x := newHarness(t) + x.meta.book.Sections[1].ListenURL += " " + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable || len(x.fetch.seen()) != 0 || x.exists("L. Leslie Brooke") { + t.Fatalf("code %q (%v), %d requests", code, err, len(x.fetch.seen())) + } +} diff --git a/internal/librivoxjob/names.go b/internal/librivoxjob/names.go index 74808c3..6861c8a 100644 --- a/internal/librivoxjob/names.go +++ b/internal/librivoxjob/names.go @@ -163,8 +163,12 @@ func AllowedURL(u *url.URL) bool { return host == "archive.org" || (strings.HasSuffix(host, ".archive.org") && len(host) > len(".archive.org")) } -// allowedRaw is AllowedURL over a URL string. +// allowedRaw is AllowedURL over a URL string, as it is requested: a padded +// one is refused here rather than checked trimmed and then sent as it is. func allowedRaw(raw string) bool { - u, err := url.Parse(strings.TrimSpace(raw)) + if raw != strings.TrimSpace(raw) { + return false + } + u, err := url.Parse(raw) return err == nil && AllowedURL(u) } From 10a1f0080ef33a3f5bbe4d424d54a595cc3edb49 Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:31:51 +0000 Subject: [PATCH 18/22] librivox.fetch: survive a meta outage, keep only audio, and key state per library A meta read that fails for want of service (5xx, a dropped connection) is now tried 4 times inside the run and then fails uncoded with the staging folder and state kept, so the next attempt resumes instead of downloading again; only meta's definite 404 stays terminal as not_found. A section is kept only when it is audio: a response whose Content-Type is a page (text/html, JSON, any non-audio type) is never written, and a whole file whose first bytes are no ID3 tag or MPEG frame sync (or the Ogg, FLAC or MP4 signature) is removed. Both are tried again like a dropped connection, then fail archive_unavailable. netguard's StreamResponse now carries the Content-Type. The state file is keyed by library and project (<library_id>-<librivox_id>.json), so a fetch of one project into a second library no longer discards the first library's staging, and Abandon only touches its own library's fetch. The lost-fold adopt path is unchanged. Part of LibriVox L6-1 (Add from LibriVox). --- PACKAGES.md | 23 ++- internal/librivoxjob/download.go | 55 ++++++ internal/librivoxjob/fake_test.go | 59 +++++-- internal/librivoxjob/librivoxjob.go | 89 ++++++---- internal/librivoxjob/librivoxjob_test.go | 205 ++++++++++++++++++++++- internal/librivoxjob/state.go | 34 ++-- internal/netguard/allow_test.go | 3 + internal/netguard/stream.go | 15 +- 8 files changed, 409 insertions(+), 74 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 47cbe81..8026a54 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -2045,6 +2045,8 @@ internal/ about the first URL and every redirect's (CheckRedirect); a refusal is blocked_host before the refused address is dialled. Nil keeps the old rule (the podcasts pass none). + StreamResponse carries the Content-Type as sent (librivox.fetch refuses + a page where a file was asked for). podcast/ pure feed.json v1 types, rooted regular-file IO with capped atomic writes, URL normalization/hash/privacy/redaction, NFC show names, stable episode paths, identity merge and adaptive polling. Spike 5 @@ -2615,7 +2617,11 @@ internal/ base (metaops.LibrivoxAudiobook: /api/v1/librivox/audiobooks?id=N& extended=1&coverart=1, LibriVox's shape; decision 5, the server's mirror is never asked) and the work (metaops.LibrivoxWork, works/{id}; - a 404 there is no failure, the project names the folder then). Every + a 404 there is no failure, the project names the folder then). meta + unavailable (5xx, a dropped connection) is never terminal: each read + is tried 4 times (5, 10, 20 s apart), then the run fails uncoded + (keepStaging) with its staging folder and state kept for the next + attempt; only meta's definite 404 ends the fetch as not_found. Every section URL must be https on archive.org or *.archive.org (AllowedURL, checked before anything is fetched or written, and on every redirect through netguard's per-request Allow; never @@ -2646,7 +2652,12 @@ internal/ bytes a later 206 would be proven to continue), a resume asks Range/If-Range (netguard.ResumeAction proves the 206; anything unproven starts the file again), renamed to its name only at the - stated size and never empty (a finished file is skipped on resume); + stated size and never empty, and only audio (a Content-Type other + than none, audio/*, octet-stream or application/ogg is never written; + a whole file whose first bytes are no ID3 tag, MPEG frame sync, OggS, + fLaC or MP4 ftyp is removed; both tried again like a dropped + connection, then archive_unavailable; a finished file is skipped on + resume); three attempts in a row that add nothing fail archive_unavailable (adding = past the MOST the partial has held, section's high-water mark, so restarts from zero that only pass the last cut don't @@ -2668,7 +2679,9 @@ internal/ sections are looked for again after the removals), ONE fsutil.RenameNoReplace staging -> book folder (a folder that raced in takes the fallback once, never replaced), then - the rescan (3 tries). State <data>/librivox/<id>.json (state.go: the + the rescan (3 tries). State <data>/librivox/<library_id>-<id>.json, + one per library and project, so fetches of one project into two + libraries never touch each other's staging (state.go: the library, parent and whether the fetch made it, destination (staging is <parent>/.audiosilo-librivox-<id>, never stored), the partials' validators, then placed + count + bytes): a resumed run @@ -2680,8 +2693,8 @@ internal/ gone and the folder or its fallback holding this project is placed (adopt), never fetched again. A stop or a lost claim keeps staging and state; a cancel, a - failure or Abandon (not on a lost claim, and only a state for the - abandoned job's own library: the state is keyed by project) removes + failure or Abandon (not on a lost claim; only the abandoned job's own + library's state) removes the staging folder, the author folder it made when empty, and the state. A lost claim never handed out again leaves its staging folder (hidden, unindexed). diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go index e44452f..c2e98b2 100644 --- a/internal/librivoxjob/download.go +++ b/internal/librivoxjob/download.go @@ -2,16 +2,19 @@ package librivoxjob import ( "bufio" + "bytes" "context" "errors" "fmt" "io" "io/fs" "math" + "mime" "net/http" "os" "path" "path/filepath" + "strings" "time" "github.com/kodestar/audiosilo-bridge/internal/fsutil" @@ -179,6 +182,11 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r default: return fail(CodeArchiveUnavailable, "archive.org answered HTTP %d for section %d of LibriVox recording %d.", resp.Status, i+1, f.p.LibrivoxID) } + if !audioType(resp.ContentType) { + // A page where a file was asked for (an error or a login page with a + // 200): never written, tried again like a dropped connection. + return &retry{size: offset, cause: "not_audio"} + } if total > maxFileBytes { return fmt.Errorf("section %d is larger than the Bridge's cap for one file", i+1) } @@ -254,6 +262,12 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r } return &retry{size: size, cause: "short"} } + if !audioHead(root, part) { + // Whole but not audio: it goes, and the section starts again. + _ = root.Remove(part) + delete(f.st.Files, name) + return &retry{cause: "not_audio"} + } if err := root.Rename(part, final); err != nil { return writeFailure(err, final) } @@ -263,6 +277,47 @@ func (h *Handler) attempt(ctx context.Context, root *os.Root, f *fetch, i int, r return nil } +// audioType reports whether a response's Content-Type may carry a section: +// none stated, audio/*, or a generic binary type. A text, HTML, JSON or other +// typed answer is a page, not the file. +func audioType(contentType string) bool { + if strings.TrimSpace(contentType) == "" { + return true + } + mt, _, err := mime.ParseMediaType(contentType) + if err != nil { + return false + } + return strings.HasPrefix(mt, "audio/") || mt == "application/octet-stream" || mt == "binary/octet-stream" || + mt == "application/ogg" +} + +// audioHead reports whether a finished section's first bytes are audio: an +// ID3 tag or an MPEG frame sync (LibriVox's MP3s), else the Ogg, FLAC or MP4 +// signature a section of another extension starts with. +func audioHead(root *os.Root, name string) bool { + file, err := root.Open(name) + if err != nil { + return false + } + defer func() { _ = file.Close() }() + head := make([]byte, 12) + n, _ := io.ReadFull(file, head) + return isAudioHead(head[:n]) +} + +func isAudioHead(b []byte) bool { + switch { + case bytes.HasPrefix(b, []byte("ID3")), bytes.HasPrefix(b, []byte("OggS")), bytes.HasPrefix(b, []byte("fLaC")): + return true + case len(b) >= 2 && b[0] == 0xFF && b[1]&0xE0 == 0xE0: + return true + case len(b) >= 8 && string(b[4:8]) == "ftyp": + return true + } + return false +} + // writeErr remembers a write's failure, so a failed copy tells the library's // fault (disk_full, not_writable) from the connection's (try again). type writeErr struct { diff --git a/internal/librivoxjob/fake_test.go b/internal/librivoxjob/fake_test.go index a043261..c9134db 100644 --- a/internal/librivoxjob/fake_test.go +++ b/internal/librivoxjob/fake_test.go @@ -2,6 +2,7 @@ package librivoxjob import ( "bytes" + "cmp" "context" "errors" "fmt" @@ -28,10 +29,24 @@ type fakeMeta struct { work metaops.LibrivoxWork workErr error reads int + // bookErrs and workErrs: each read pops one first (nil: answer as usual). + bookErrs, workErrs []error +} + +func pop(q *[]error) error { + if len(*q) == 0 { + return nil + } + err := (*q)[0] + *q = (*q)[1:] + return err } func (m *fakeMeta) LibrivoxAudiobook(_ context.Context, id int) (metaops.LibrivoxBook, error) { m.reads++ + if err := pop(&m.bookErrs); err != nil { + return metaops.LibrivoxBook{}, err + } if m.bookErr != nil { return metaops.LibrivoxBook{}, m.bookErr } @@ -42,12 +57,17 @@ func (m *fakeMeta) LibrivoxAudiobook(_ context.Context, id int) (metaops.Librivo } func (m *fakeMeta) LibrivoxWork(context.Context, string) (metaops.LibrivoxWork, error) { + if err := pop(&m.workErrs); err != nil { + return metaops.LibrivoxWork{}, err + } return m.work, m.workErr } // remote is one file the fake archive.org serves. type remote struct { - body []byte + body []byte + // ctype is the Content-Type sent ("" is audio/mpeg). + ctype string etag string status int // an error status instead of the file // redirect: the file is served from this URL, which Allow is asked about @@ -131,7 +151,7 @@ func (f *fakeFetcher) Stream(_ context.Context, in netguard.StreamRequest) (*net if r.status != 0 { return &netguard.StreamResponse{Status: r.status, Body: io.NopCloser(bytes.NewReader(nil)), ContentLength: 0}, nil } - resp := &netguard.StreamResponse{Status: http.StatusOK, ETag: r.etag, ContentLength: int64(len(r.body))} + resp := &netguard.StreamResponse{Status: http.StatusOK, ETag: r.etag, ContentLength: int64(len(r.body)), ContentType: cmp.Or(r.ctype, "audio/mpeg")} body := r.body // Range only with a strong ETag, as netguard sends it. if in.Offset > 0 && netguard.StrongETag(in.ETag) && in.ETag == r.etag && in.Offset < int64(len(r.body)) { @@ -166,6 +186,11 @@ func (f *fakeFetcher) Fetch(_ context.Context, in netguard.Request) (*netguard.R return &netguard.Response{Status: http.StatusOK, Body: r.body}, nil } +// mp3 is an n-byte section body: an ID3 header, then fill. +func mp3(fill byte, n int) []byte { + return append([]byte("ID3"), bytes.Repeat([]byte{fill}, n-3)...) +} + // fakeServer is the server link: the library's books and the rescan. type fakeServer struct { mu sync.Mutex @@ -201,20 +226,34 @@ type fakeLibs struct { mount string writable bool kind string + // second, when set, is library 4's mount: a second books library. + second string +} + +// mountOf is library id's mount, "" for none. +func (l *fakeLibs) mountOf(id int64) string { + switch { + case id == 3: + return l.mount + case id == 4 && l.second != "": + return l.second + } + return "" } func (l *fakeLibs) Library(id int64) (link.Library, bool) { - if id != 3 { + if l.mountOf(id) == "" { return link.Library{}, false } - return link.Library{ID: 3, Name: "Books", Root: "/srv/books", Kind: l.kind}, true + return link.Library{ID: id, Name: "Books", Root: "/srv/books" + strconv.FormatInt(id, 10), Kind: l.kind}, true } func (l *fakeLibs) Status(id int64) (link.LibraryStatus, bool) { - if id != 3 { + mount := l.mountOf(id) + if mount == "" { return link.LibraryStatus{}, false } - return link.LibraryStatus{LibraryID: 3, MountPath: l.mount, Mapped: true, Writable: l.writable}, true + return link.LibraryStatus{LibraryID: id, MountPath: mount, Mapped: true, Writable: l.writable}, true } // harness is a handler over the fakes, a library folder and a state folder. @@ -266,15 +305,15 @@ func newHarness(t *testing.T) *harness { Recordings: []metaops.LibrivoxRecording{{ID: testRec, LibrivoxID: testID, ArchiveID: testItem}}}, } x.fetch = &fakeFetcher{files: map[string]*remote{ - sectionURL(1): {body: bytes.Repeat([]byte("1"), 1000), etag: `"etag-1"`}, - sectionURL(2): {body: bytes.Repeat([]byte("2"), 2000), etag: `"etag-2"`}, - sectionURL(3): {body: bytes.Repeat([]byte("3"), 3000), etag: `"etag-3"`}, + sectionURL(1): {body: mp3('1', 1000), etag: `"etag-1"`}, + sectionURL(2): {body: mp3('2', 2000), etag: `"etag-2"`}, + sectionURL(3): {body: mp3('3', 3000), etag: `"etag-3"`}, coverURL: {body: jpeg}, }} x.srv = &fakeServer{} x.libs = &fakeLibs{mount: x.mount, writable: true} x.h = New(Options{Meta: x.meta, Fetcher: x.fetch, Libraries: x.libs, StateDir: x.stateDir, - Mount: func(int64, string) (string, bool) { return x.mount, true }, + Mount: func(id int64, _ string) (string, bool) { return x.libs.mountOf(id), true }, Server: func() Server { return x.srv }, Free: func(string) (int64, error) { return x.free, nil }, Wait: func(ctx context.Context, _ time.Duration) error { return ctx.Err() }, diff --git a/internal/librivoxjob/librivoxjob.go b/internal/librivoxjob/librivoxjob.go index 775658c..687b3e1 100644 --- a/internal/librivoxjob/librivoxjob.go +++ b/internal/librivoxjob/librivoxjob.go @@ -197,9 +197,9 @@ const mappingPoll = time.Second // Abandon removes the staging folder of a job the runner ended without // running it (cancelled before it resumed). A lost claim keeps it: the server -// hands the job out again and the next claim resumes it. A placed book stays, -// and so does a state for another library: the state is the project's, not -// the job's, and that one is a later fetch's. +// hands the job out again and the next claim resumes it. A placed book stays. +// The state is the job's library's alone, so a fetch of the same project +// into another library is never touched. func (h *Handler) Abandon(ctx context.Context, job jobs.Job) { if jobs.Lost(ctx) { return @@ -208,22 +208,16 @@ func (h *Handler) Abandon(ctx context.Context, job jobs.Job) { if err != nil { return } - st := h.loadState(p.LibrivoxID) - if st == nil || st.Placed || st.LibraryID != p.LibraryID { + st := h.loadState(p.LibraryID, p.LibrivoxID) + if st == nil || st.Placed { return } - h.discard(p.LibrivoxID, st) -} - -// discard removes a fetch's staging folder from the library its state names -// (best effort) and the state. -func (h *Handler) discard(id int, st *state) { if lib, ok := h.o.Libraries.Library(st.LibraryID); ok { if mount, _ := h.o.Mount(lib.ID, lib.Root); mount != "" { - h.removeStaging(mount, id, st) + h.removeStaging(mount, p.LibrivoxID, st) } } - h.dropState(id) + h.dropState(st.LibraryID, p.LibrivoxID) } // Run fetches the recording: manifest, download, cover, place, rescan. @@ -279,29 +273,21 @@ func (h *Handler) run(ctx context.Context, job jobs.Job, p Params, rep *jobs.Rep if err != nil { return nil, err } - f := &fetch{p: p, lib: lib, mount: mount, rep: rep, st: h.loadState(p.LibrivoxID)} - if f.st != nil && f.st.LibraryID != lib.ID { - // An earlier fetch of the project for another library: its staging - // folder goes (best effort; a book it placed stays), and this one plans - // afresh. - if !f.st.Placed { - h.discard(p.LibrivoxID, f.st) - } - f.st = nil - } + f := &fetch{p: p, lib: lib, mount: mount, rep: rep, st: h.loadState(lib.ID, p.LibrivoxID)} defer func() { // A stop the next start resumes, and a lost claim the server hands out // again, keep the staging folder and the state; a cancel or a failure // before the book was placed removes both. A placed book stays, and its // state with it when the rescan failed (the retry goes on from there). - if err == nil || f.st == nil || f.st.Placed { + var keep *keepStaging + if err == nil || f.st == nil || f.st.Placed || errors.As(err, &keep) { return } if ctx.Err() != nil && (jobs.Resumes(ctx) || jobs.Lost(ctx)) { return } h.removeStaging(f.mount, p.LibrivoxID, f.st) - h.dropState(p.LibrivoxID) + h.dropState(lib.ID, p.LibrivoxID) }() // An earlier run placed the book and then failed (the rescan): while its @@ -313,12 +299,12 @@ func (h *Handler) run(ctx context.Context, job jobs.Job, p Params, rep *jobs.Rep if h.holdsProject(mount, f.st.Dest, p.LibrivoxID) { return h.finish(ctx, f, srv) } - h.dropState(p.LibrivoxID) + h.dropState(lib.ID, p.LibrivoxID) f.st = nil } h.report(rep, jobs.Progress{Step: StepManifest, Message: "Reading the recording from the metadata database"}) - book, work, err := h.manifest(ctx, p) + book, work, err := h.manifest(ctx, p, rep) if err != nil { return nil, err } @@ -359,14 +345,19 @@ func (h *Handler) run(ctx context.Context, job jobs.Job, p Params, rep *jobs.Rep // manifest reads the project's manifest and its work from meta. A work meta // doesn't have (another meta base than the server's, decision 5) is no // failure: the project names the folder then. -func (h *Handler) manifest(ctx context.Context, p Params) (metaops.LibrivoxBook, *metaops.LibrivoxWork, error) { +// +// meta unavailable (a 5xx, a dropped connection) is never terminal: each read +// is tried metaAttempts times, and still unavailable the run fails with a +// keepStaging error, which keeps the staging folder and the state for the +// next attempt. Only meta's definite 404 (not_found) ends the fetch. +func (h *Handler) manifest(ctx context.Context, p Params, rep *jobs.Reporter) (metaops.LibrivoxBook, *metaops.LibrivoxWork, error) { metadataOff := func() error { return fail(CodeMetadataOff, "The Bridge has no metadata service set (metadata.base_url), so it can't read LibriVox recordings.") } if h.o.Meta == nil { return metaops.LibrivoxBook{}, nil, metadataOff() } - book, err := h.o.Meta.LibrivoxAudiobook(ctx, p.LibrivoxID) + book, err := metaRead(ctx, h, rep, func() (metaops.LibrivoxBook, error) { return h.o.Meta.LibrivoxAudiobook(ctx, p.LibrivoxID) }) switch { case errors.Is(err, metaops.ErrDisabled): return book, nil, metadataOff() @@ -382,7 +373,7 @@ func (h *Handler) manifest(ctx context.Context, p Params) (metaops.LibrivoxBook, return book, nil, fail(CodeArchiveUnavailable, "Section %d of LibriVox recording %d isn't a file on archive.org, the only place the Bridge downloads LibriVox from.", i+1, p.LibrivoxID) } } - w, err := h.o.Meta.LibrivoxWork(ctx, p.WorkID) + w, err := metaRead(ctx, h, rep, func() (metaops.LibrivoxWork, error) { return h.o.Meta.LibrivoxWork(ctx, p.WorkID) }) switch { case errors.Is(err, metaops.ErrNotFound): return book, nil, nil @@ -398,6 +389,42 @@ func (h *Handler) manifest(ctx context.Context, p Params) (metaops.LibrivoxBook, return book, &w, nil } +// metaAttempts bounds one meta read while meta is unavailable; metaWait is +// the first wait between two, doubled each time (5 s, 10 s, 20 s). +const ( + metaAttempts = 4 + metaWait = 5 * time.Second +) + +// keepStaging is a run's failure that leaves the staging folder and the state +// for the next attempt (meta unavailable): nothing the fetch holds is wrong. +type keepStaging struct{ err error } + +func (k *keepStaging) Error() string { return k.err.Error() } +func (k *keepStaging) Unwrap() error { return k.err } + +// metaRead is read, tried again while meta is unavailable +// (metaops.ErrUnavailable); still unavailable after metaAttempts, the error is +// a keepStaging. Any other answer (a 404, metadata off, the context) is +// returned at once. +func metaRead[T any](ctx context.Context, h *Handler, rep *jobs.Reporter, read func() (T, error)) (T, error) { + wait := metaWait + for attempt := 1; ; attempt++ { + v, err := read() + if err == nil || !errors.Is(err, metaops.ErrUnavailable) || ctx.Err() != nil { + return v, err + } + if attempt == metaAttempts { + return v, &keepStaging{err: err} + } + h.report(rep, jobs.Progress{Step: StepManifest, Message: "The metadata database isn't answering: trying again"}) + if werr := h.o.Wait(ctx, wait); werr != nil { + return v, werr + } + wait *= 2 + } +} + // archiveIDOf is the project's Internet Archive item: the recording's in the // work, else the feed's url_iarchive. func archiveIDOf(p Params, book metaops.LibrivoxBook, work *metaops.LibrivoxWork) string { @@ -756,7 +783,7 @@ func (h *Handler) finish(ctx context.Context, f *fetch, srv Server) (*Result, er } h.o.Log.Info("librivox: added", "librivox_id", f.p.LibrivoxID, "library", f.lib.ID, "path", bookPath, "files", f.st.Count, "bytes", f.st.Bytes) - h.dropState(f.p.LibrivoxID) + h.dropState(f.lib.ID, f.p.LibrivoxID) return &Result{LibraryID: f.lib.ID, Path: bookPath, LibrivoxID: f.p.LibrivoxID, Files: f.st.Count, Bytes: f.st.Bytes}, nil } diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index c6895f6..5d2fcaa 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -593,13 +593,18 @@ func TestStateValid(t *testing.T) { } x := newHarness(t) _ = os.MkdirAll(x.stateDir, 0o700) - _ = os.WriteFile(x.h.statePath(192), []byte(`{"library_id":3,"parent":"..","dest":"../T"}`), 0o600) - if x.h.loadState(192) != nil { + _ = os.WriteFile(x.h.statePath(3, 192), []byte(`{"library_id":3,"parent":"..","dest":"../T"}`), 0o600) + if x.h.loadState(3, 192) != nil { t.Fatal("an invalid state loaded") } - if _, err := os.Stat(x.h.statePath(192)); err == nil { + if _, err := os.Stat(x.h.statePath(3, 192)); err == nil { t.Fatal("an invalid state stayed") } + // A state file naming another library than its own is never followed. + _ = os.WriteFile(x.h.statePath(3, 192), []byte(`{"library_id":4,"parent":"A","dest":"A/T"}`), 0o600) + if x.h.loadState(3, 192) != nil { + t.Fatal("another library's state loaded") + } } // A resumed job whose library isn't writable now parks not_writable and goes @@ -656,9 +661,17 @@ func TestFetchTransientStatuses(t *testing.T) { type flakyFetcher struct { *fakeFetcher statuses map[string][]int + // html: that many 200 answers of an HTML page before the file. + html map[string]int } func (f *flakyFetcher) Stream(ctx context.Context, in netguard.StreamRequest) (*netguard.StreamResponse, error) { + if f.html[in.URL] > 0 { + f.html[in.URL]-- + page := "<html>not found</html>" + return &netguard.StreamResponse{Status: 200, Body: io.NopCloser(strings.NewReader(page)), ContentLength: int64(len(page)), + ContentType: "text/html"}, nil + } if q := f.statuses[in.URL]; len(q) > 0 { f.statuses[in.URL] = q[1:] return &netguard.StreamResponse{Status: q[0], Body: io.NopCloser(strings.NewReader("")), RetryAfterSeconds: 1}, nil @@ -755,7 +768,7 @@ func TestFetchRestartDropsPartialBeforeValidators(t *testing.T) { } // The file changed on archive.org: the resume's If-Range gets the whole // new file under a new ETag. - x.fetch.files[sectionURL(2)] = &remote{body: bytes.Repeat([]byte("n"), 2000), etag: `"etag-2b"`} + x.fetch.files[sectionURL(2)] = &remote{body: mp3('n', 2000), etag: `"etag-2b"`} checked := false x.h.o.Free = func(string) (int64, error) { if !checked { // the restarted section's attempt, before its state is saved @@ -833,7 +846,7 @@ func TestFetchAdoptsUnrecordedPlacement(t *testing.T) { if _, err := x.run(context.Background(), false); err == nil { t.Fatal("the first run's rescan didn't fail") } - st := x.h.loadState(testID) + st := x.h.loadState(3, testID) if st == nil || !st.Placed { t.Fatalf("state %+v", st) } @@ -865,7 +878,7 @@ func TestAbandonLeavesAnotherLibrarysFetch(t *testing.T) { other := x.job(true) other.Params = bytes.Replace(other.Params, []byte(`"library_id":3`), []byte(`"library_id":4`), 1) x.h.Abandon(jobs.AbandonContext(context.Background(), false), other) - if !x.exists(stagingDir) || x.h.loadState(testID) == nil { + if !x.exists(stagingDir) || x.h.loadState(3, testID) == nil { t.Fatal("Abandon removed another library's fetch") } } @@ -888,3 +901,183 @@ func TestFetchPaddedURLRefused(t *testing.T) { t.Fatalf("code %q (%v), %d requests", code, err, len(x.fetch.seen())) } } + +// meta unavailable for a while is tried again inside the run, for the +// manifest and the work alike. +func TestFetchMetaUnavailableRetried(t *testing.T) { + x := newHarness(t) + u := metaops.ErrUnavailable + x.meta.bookErrs = []error{u, u} + x.meta.workErrs = []error{u} + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Path != bookDir { + t.Fatalf("%#v %v", out, err) + } + if x.meta.reads != 3 { + t.Fatalf("%d manifest reads", x.meta.reads) + } +} + +// meta unavailable past the retries on a resumed run fails without a code and +// keeps the staging folder and the state: the next attempt resumes, never +// fetching a finished section again. A definite 404 still ends the fetch. +func TestFetchMetaUnavailableKeepsStaging(t *testing.T) { + for _, which := range []string{"manifest", "work"} { + t.Run(which, func(t *testing.T) { + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = stop + _, _ = x.run(ctx, false) + if !x.exists(stagingDir + "/01 - Johnny Crow's Garden.mp3") { + t.Fatal("no staged section") + } + u := metaops.ErrUnavailable + if which == "manifest" { + x.meta.bookErr = u + } else { + x.meta.workErr = u + } + out, err := x.run(context.Background(), true) + if err == nil || !errors.Is(err, metaops.ErrUnavailable) { + t.Fatalf("resumed run: %v", err) + } + if code := failureCode(t, out, err); code != "" { + t.Fatalf("meta unavailable was coded %q", code) + } + if !x.exists(stagingDir+"/01 - Johnny Crow's Garden.mp3") || x.h.loadState(3, testID) == nil { + t.Fatal("meta unavailable removed the staging folder or the state") + } + x.meta.bookErr, x.meta.workErr = nil, nil + before := len(x.fetch.seen()) + if _, err := x.run(context.Background(), true); err != nil { + t.Fatal(err) + } + for _, r := range x.fetch.seen()[before:] { + if r.url == sectionURL(1) { + t.Fatal("a finished section was fetched again") + } + } + }) + } + x := newHarness(t) + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = stop + _, _ = x.run(ctx, false) + x.meta.bookErr = metaops.ErrNotFound + out, err := x.run(context.Background(), true) + if code := failureCode(t, out, err); code != CodeNotFound { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) || x.h.loadState(3, testID) != nil { + t.Fatal("not_found kept the staging folder") + } +} + +// A section that isn't audio (a page sent with a 200, or bytes that are +// neither an ID3 tag nor an MPEG frame) is never kept: it is tried again, +// then archive_unavailable, and nothing is placed. +func TestFetchRefusesNonAudio(t *testing.T) { + for name, set := range map[string]func(r *remote){ + "html type": func(r *remote) { r.ctype = "text/html; charset=utf-8" }, + "json type": func(r *remote) { r.ctype = "application/json" }, + "html bytes": func(r *remote) { r.body = append([]byte("<!DOCTYPE html>"), r.body[15:]...) }, + "text bytes": func(r *remote) { r.body = bytes.Repeat([]byte("x"), len(r.body)) }, + } { + t.Run(name, func(t *testing.T) { + x := newHarness(t) + set(x.fetch.files[sectionURL(2)]) + out, err := x.run(context.Background(), false) + if code := failureCode(t, out, err); code != CodeArchiveUnavailable { + t.Fatalf("code %q (%v)", code, err) + } + if x.exists(stagingDir) || x.exists(bookDir) { + t.Fatal("a non-audio section was kept or placed") + } + }) + } + // Once: tried again, and the real file then is kept. + x := newHarness(t) + x.h.o.Fetcher = &flakyFetcher{fakeFetcher: x.fetch, html: map[string]int{sectionURL(2): 1}} + out, err := x.run(context.Background(), false) + if err != nil || out.(*Result).Bytes != 6014 { + t.Fatalf("%#v %v", out, err) + } +} + +func TestAudioChecks(t *testing.T) { + for ct, want := range map[string]bool{ + "": true, "audio/mpeg": true, "audio/ogg; codecs=vorbis": true, "application/octet-stream": true, + "application/ogg": true, "text/html": false, "text/plain; charset=utf-8": false, "application/json": false, + "image/jpeg": false, ";;bad": false, + } { + if got := audioType(ct); got != want { + t.Errorf("audioType(%q) = %v", ct, got) + } + } + for head, want := range map[string]bool{ + "ID3\x04": true, "\xFF\xFB\x90": true, "\xFF\xF3": true, "OggS": true, "fLaC": true, "\x00\x00\x00\x20ftypM4A ": true, + "<html>": false, "": false, "\xFF\x00": false, "x": false, + } { + if got := isAudioHead([]byte(head)); got != want { + t.Errorf("isAudioHead(%q) = %v", head, got) + } + } +} + +// Two libraries fetching one project keep their own state and staging: a run +// into one never discards the other's, and Abandon of one leaves the other. +func TestFetchTwoLibrariesIndependent(t *testing.T) { + x := newHarness(t) + other := t.TempDir() + x.libs.second = other + ctx, stop := context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].cut = []int{100} + x.fetch.files[sectionURL(2)].onCut = stop + _, _ = x.run(ctx, false) // library 3 stops mid-download + if !x.exists(stagingDir) { + t.Fatal("no staging in library 3") + } + // Library 4 fetches the whole project. + job4 := x.job(false) + lib4 := int64(4) + job4.LibraryID = &lib4 + job4.Params = bytes.Replace(job4.Params, []byte(`"library_id":3`), []byte(`"library_id":4`), 1) + out, err := x.h.Run(context.Background(), job4, jobs.LocalReporter()) + if err != nil || out.(*Result).LibraryID != 4 { + t.Fatalf("library 4: %#v %v", out, err) + } + if _, err := os.Stat(path.Join(other, bookDir, "01 - Johnny Crow's Garden.mp3")); err != nil { + t.Fatal("library 4's book isn't there") + } + if !x.exists(stagingDir+"/01 - Johnny Crow's Garden.mp3") || x.h.loadState(3, testID) == nil { + t.Fatal("library 4's fetch discarded library 3's staging") + } + // Library 4 stops mid-download too; Abandon of library 3's job leaves it. + x.fetch.files[sectionURL(2)].cut = []int{100} + ctx, stop = context.WithCancel(context.Background()) + x.fetch.files[sectionURL(2)].onCut = stop + if err := os.RemoveAll(path.Join(other, bookDir)); err != nil { + t.Fatal(err) + } + _, _ = x.h.Run(ctx, job4, jobs.LocalReporter()) + staging4 := path.Join(other, stagingDir) + if _, err := os.Stat(staging4); err != nil { + t.Fatal("no staging in library 4") + } + x.h.Abandon(jobs.AbandonContext(context.Background(), false), x.job(true)) + if x.exists(stagingDir) { + t.Fatal("Abandon left library 3's staging") + } + if _, err := os.Stat(staging4); err != nil || x.h.loadState(4, testID) == nil { + t.Fatal("Abandon of library 3's job removed library 4's fetch") + } + // Library 3 resumes from scratch independently and finishes. + if _, err := x.run(context.Background(), false); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(staging4); err != nil { + t.Fatal("library 3's run removed library 4's staging") + } +} diff --git a/internal/librivoxjob/state.go b/internal/librivoxjob/state.go index 563c885..7c1ece0 100644 --- a/internal/librivoxjob/state.go +++ b/internal/librivoxjob/state.go @@ -14,7 +14,9 @@ import ( ) // state is a fetch's plan and progress, kept in the Bridge's data -// (<StateDir>/<librivox_id>.json) from the moment it picks the folder: a +// (<StateDir>/<library_id>-<librivox_id>.json, one per library and project, so +// fetches of one project into two libraries never touch each other's staging) +// from the moment it picks the folder: a // resumed run (a restart, a lost claim handed out again) goes on with the same // staging folder and destination instead of planning again against a library // that changed meanwhile, Abandon finds the staging folder to remove, and once @@ -51,22 +53,22 @@ func stagingName(id int) string { return ".audiosilo-librivox-" + strconv.Itoa(i // staging is the project's staging folder: <parent>/.audiosilo-librivox-<id>. func (st *state) staging(id int) string { return path.Join(st.Parent, stagingName(id)) } -// statePath is the project's state file. -func (h *Handler) statePath(id int) string { - return filepath.Join(h.o.StateDir, strconv.Itoa(id)+".json") +// statePath is the state file of the project's fetch into library lib. +func (h *Handler) statePath(lib int64, id int) string { + return filepath.Join(h.o.StateDir, strconv.FormatInt(lib, 10)+"-"+strconv.Itoa(id)+".json") } -// loadState reads the project's state, nil when there is none. One that -// can't be read, or whose paths aren't clean library paths, goes: the run -// plans afresh. -func (h *Handler) loadState(id int) *state { - p := h.statePath(id) +// loadState reads the state of the project's fetch into library lib, nil when +// there is none. One that can't be read, that names another library, or whose +// paths aren't clean library paths, goes: the run plans afresh. +func (h *Handler) loadState(lib int64, id int) *state { + p := h.statePath(lib, id) b, err := os.ReadFile(p) //nolint:gosec // G304: the job's own state in the Bridge's data if err != nil { return nil } var st state - if json.Unmarshal(b, &st) != nil || !st.valid() { + if json.Unmarshal(b, &st) != nil || st.LibraryID != lib || !st.valid() { _ = os.Remove(p) return nil } @@ -89,7 +91,7 @@ func (st *state) valid() bool { return err == nil && clean == st.Dest && path.Dir(st.Dest) == parent } -// saveState writes the project's state. +// saveState writes the state of the project's fetch into st's library. func (h *Handler) saveState(id int, st *state) error { if err := os.MkdirAll(h.o.StateDir, 0o700); err != nil { return err @@ -98,12 +100,12 @@ func (h *Handler) saveState(id int, st *state) error { if err != nil { return err } - return fsutil.WriteFileAtomic(h.statePath(id), b, 0o600) + return fsutil.WriteFileAtomic(h.statePath(st.LibraryID, id), b, 0o600) } -// dropState removes the project's state. -func (h *Handler) dropState(id int) { - if err := os.Remove(h.statePath(id)); err != nil && !errors.Is(err, fs.ErrNotExist) { - h.o.Log.Warn("librivox: removing a fetch's state failed", "librivox_id", id, "err", err) +// dropState removes the state of the project's fetch into library lib. +func (h *Handler) dropState(lib int64, id int) { + if err := os.Remove(h.statePath(lib, id)); err != nil && !errors.Is(err, fs.ErrNotExist) { + h.o.Log.Warn("librivox: removing a fetch's state failed", "library", lib, "librivox_id", id, "err", err) } } diff --git a/internal/netguard/allow_test.go b/internal/netguard/allow_test.go index 5b82099..c267678 100644 --- a/internal/netguard/allow_test.go +++ b/internal/netguard/allow_test.go @@ -57,6 +57,9 @@ func TestRequestAllowFirstURLAndRedirects(t *testing.T) { if string(b) != "body" { t.Fatalf("%s: %q", p, b) } + if !strings.HasPrefix(r.ContentType, "text/plain") { + t.Fatalf("%s: content type %q", p, r.ContentType) + } f, err := c.Fetch(context.Background(), Request{URL: base + p, MaxBytes: 100, Allow: allow}) if err != nil || string(f.Body) != "body" { t.Fatal(p, err) diff --git a/internal/netguard/stream.go b/internal/netguard/stream.go index 790a37f..5c94f79 100644 --- a/internal/netguard/stream.go +++ b/internal/netguard/stream.go @@ -22,11 +22,14 @@ type StreamRequest struct { Allow func(*url.URL) bool } type StreamResponse struct { - Status int - Body io.ReadCloser - ContentLength int64 - ContentRange string - Encoding string + Status int + Body io.ReadCloser + ContentLength int64 + ContentRange string + Encoding string + // ContentType is the response's Content-Type as sent (librivox.fetch + // refuses a page where a file was asked for). + ContentType string ETag string RetryAfterSeconds int } @@ -72,7 +75,7 @@ func (c *Client) Stream(ctx context.Context, in StreamRequest) (*StreamResponse, } timer := time.AfterFunc(c.stall, cancel) body := &streamBody{progressReader: progressReader{r: resp.Body, timer: timer, stall: c.stall}, body: resp.Body, cancel: cancel} - return &StreamResponse{Status: resp.StatusCode, Body: body, ContentLength: resp.ContentLength, ContentRange: resp.Header.Get("Content-Range"), Encoding: strings.ToLower(strings.TrimSpace(resp.Header.Get("Content-Encoding"))), ETag: resp.Header.Get("ETag"), RetryAfterSeconds: retryAfter(resp.Header.Get("Retry-After"), time.Now())}, nil + return &StreamResponse{Status: resp.StatusCode, Body: body, ContentLength: resp.ContentLength, ContentRange: resp.Header.Get("Content-Range"), Encoding: strings.ToLower(strings.TrimSpace(resp.Header.Get("Content-Encoding"))), ContentType: resp.Header.Get("Content-Type"), ETag: resp.Header.Get("ETag"), RetryAfterSeconds: retryAfter(resp.Header.Get("Retry-After"), time.Now())}, nil } // streamBody resets the stall timer on progress (progressReader) and ends the From 1d32f9bc37d4988503254a51e97d43acd8315472 Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:34:48 +0000 Subject: [PATCH 19/22] Drop the librivox Abandon test that per-library state made vacuous With state keyed per library and project, Abandon of a library-4 job finds no state at all, so the old test passed without exercising its claim. TestFetchTwoLibrariesIndependent covers Abandon leaving the other library's fetch. Also re-wraps the state comment. Part of LibriVox L6. --- internal/librivoxjob/librivoxjob_test.go | 16 ---------------- internal/librivoxjob/state.go | 11 +++++------ 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index 5d2fcaa..6475123 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -867,22 +867,6 @@ func TestFetchAdoptsUnrecordedPlacement(t *testing.T) { } } -// Abandon leaves a state that is another library's fetch of the project: the -// state is the project's, and the abandoned job never wrote that one. -func TestAbandonLeavesAnotherLibrarysFetch(t *testing.T) { - x := newHarness(t) - ctx, stop := context.WithCancel(context.Background()) - x.fetch.files[sectionURL(2)].cut = []int{100} - x.fetch.files[sectionURL(2)].onCut = stop - _, _ = x.run(ctx, false) - other := x.job(true) - other.Params = bytes.Replace(other.Params, []byte(`"library_id":3`), []byte(`"library_id":4`), 1) - x.h.Abandon(jobs.AbandonContext(context.Background(), false), other) - if !x.exists(stagingDir) || x.h.loadState(3, testID) == nil { - t.Fatal("Abandon removed another library's fetch") - } -} - // A padded URL is refused as it stands: the check and the request see one // spelling. func TestFetchPaddedURLRefused(t *testing.T) { diff --git a/internal/librivoxjob/state.go b/internal/librivoxjob/state.go index 7c1ece0..7b13ae8 100644 --- a/internal/librivoxjob/state.go +++ b/internal/librivoxjob/state.go @@ -16,12 +16,11 @@ import ( // state is a fetch's plan and progress, kept in the Bridge's data // (<StateDir>/<library_id>-<librivox_id>.json, one per library and project, so // fetches of one project into two libraries never touch each other's staging) -// from the moment it picks the folder: a -// resumed run (a restart, a lost claim handed out again) goes on with the same -// staging folder and destination instead of planning again against a library -// that changed meanwhile, Abandon finds the staging folder to remove, and once -// the folder is placed it is the placement marker (a failed rescan retries -// from the rescan, decision 6). +// from the moment it picks the folder: a resumed run (a restart, a lost claim +// handed out again) goes on with the same staging folder and destination +// instead of planning again against a library that changed meanwhile, Abandon +// finds the staging folder to remove, and once the folder is placed it is the +// placement marker (a failed rescan retries from the rescan, decision 6). type state struct { LibraryID int64 `json:"library_id"` // Parent is the folder the book goes in ("" for the library's root), and From a076bb93a87601e26c899f32b4184f85b0b4a1cc Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:41:00 +0000 Subject: [PATCH 20/22] Accept an audio Content-Type with a malformed parameter mime.ParseMediaType returns the media type beside ErrInvalidMediaParameter for a header such as "audio/mpeg; charset". Treating that error as not audio refused a valid section on every attempt and ended the fetch archive_unavailable, removing its staging. Part of LibriVox L6. --- internal/librivoxjob/download.go | 4 +++- internal/librivoxjob/librivoxjob_test.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go index c2e98b2..2f9a5d3 100644 --- a/internal/librivoxjob/download.go +++ b/internal/librivoxjob/download.go @@ -284,8 +284,10 @@ func audioType(contentType string) bool { if strings.TrimSpace(contentType) == "" { return true } + // A malformed parameter ("audio/mpeg; charset") still names the type: + // ParseMediaType returns it beside ErrInvalidMediaParameter. mt, _, err := mime.ParseMediaType(contentType) - if err != nil { + if err != nil && !errors.Is(err, mime.ErrInvalidMediaParameter) { return false } return strings.HasPrefix(mt, "audio/") || mt == "application/octet-stream" || mt == "binary/octet-stream" || diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index 6475123..b4f4b16 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -994,7 +994,7 @@ func TestAudioChecks(t *testing.T) { for ct, want := range map[string]bool{ "": true, "audio/mpeg": true, "audio/ogg; codecs=vorbis": true, "application/octet-stream": true, "application/ogg": true, "text/html": false, "text/plain; charset=utf-8": false, "application/json": false, - "image/jpeg": false, ";;bad": false, + "image/jpeg": false, ";;bad": false, "audio/mpeg; charset": true, "text/html; charset": false, } { if got := audioType(ct); got != want { t.Errorf("audioType(%q) = %v", ct, got) From 44a0014310651ab4589d6b7908edd0ae7028fe57 Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:42:40 +0000 Subject: [PATCH 21/22] Skip leading zero padding when sniffing a LibriVox section A section whose MP3 frames or ID3 tag follow a few zero bytes failed the audio check on every attempt, ended archive_unavailable and lost its staging. The sniff now skips zero bytes within the first 4 KB before looking for the ID3, MPEG sync, Ogg or FLAC signature; MP4's ftyp is checked first, since its box size leads with zeros. All zeros or a page after padding is still refused. Part of LibriVox L6. --- PACKAGES.md | 5 +++-- internal/librivoxjob/download.go | 16 ++++++++++++---- internal/librivoxjob/librivoxjob_test.go | 1 + 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 8026a54..6a16563 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -2654,8 +2654,9 @@ internal/ unproven starts the file again), renamed to its name only at the stated size and never empty, and only audio (a Content-Type other than none, audio/*, octet-stream or application/ogg is never written; - a whole file whose first bytes are no ID3 tag, MPEG frame sync, OggS, - fLaC or MP4 ftyp is removed; both tried again like a dropped + a whole file whose first bytes (past zero padding in its first 4 KB) + are no ID3 tag, MPEG frame sync, OggS, fLaC or MP4 ftyp is removed; + both tried again like a dropped connection, then archive_unavailable; a finished file is skipped on resume); three attempts in a row that add nothing fail archive_unavailable diff --git a/internal/librivoxjob/download.go b/internal/librivoxjob/download.go index 2f9a5d3..5c4343e 100644 --- a/internal/librivoxjob/download.go +++ b/internal/librivoxjob/download.go @@ -296,26 +296,34 @@ func audioType(contentType string) bool { // audioHead reports whether a finished section's first bytes are audio: an // ID3 tag or an MPEG frame sync (LibriVox's MP3s), else the Ogg, FLAC or MP4 -// signature a section of another extension starts with. +// signature a section of another extension starts with. Zero bytes before +// the tag or the first frame (padding some encoders leave) are skipped within +// the first headWindow bytes. func audioHead(root *os.Root, name string) bool { file, err := root.Open(name) if err != nil { return false } defer func() { _ = file.Close() }() - head := make([]byte, 12) + head := make([]byte, headWindow) n, _ := io.ReadFull(file, head) return isAudioHead(head[:n]) } +// headWindow bounds how far audioHead looks past leading zero padding. +const headWindow = 4096 + func isAudioHead(b []byte) bool { + if len(b) >= 8 && string(b[4:8]) == "ftyp" { + // MP4's box size leads with zero bytes: checked before the padding goes. + return true + } + b = bytes.TrimLeft(b, "\x00") switch { case bytes.HasPrefix(b, []byte("ID3")), bytes.HasPrefix(b, []byte("OggS")), bytes.HasPrefix(b, []byte("fLaC")): return true case len(b) >= 2 && b[0] == 0xFF && b[1]&0xE0 == 0xE0: return true - case len(b) >= 8 && string(b[4:8]) == "ftyp": - return true } return false } diff --git a/internal/librivoxjob/librivoxjob_test.go b/internal/librivoxjob/librivoxjob_test.go index b4f4b16..a6c9922 100644 --- a/internal/librivoxjob/librivoxjob_test.go +++ b/internal/librivoxjob/librivoxjob_test.go @@ -1003,6 +1003,7 @@ func TestAudioChecks(t *testing.T) { for head, want := range map[string]bool{ "ID3\x04": true, "\xFF\xFB\x90": true, "\xFF\xF3": true, "OggS": true, "fLaC": true, "\x00\x00\x00\x20ftypM4A ": true, "<html>": false, "": false, "\xFF\x00": false, "x": false, + "\x00\x00\x00\xFF\xFB\x90": true, "\x00\x00ID3\x03": true, "\x00\x00\x00\x00": false, "\x00\x00<!DO": false, } { if got := isAudioHead([]byte(head)); got != want { t.Errorf("isAudioHead(%q) = %v", head, got) From 1213c7407d4024f2d3d8d70aff86bac4c173d7ce Mon Sep 17 00:00:00 2001 From: KodeStar <kode@audiosilo.app> Date: Fri, 13 Nov 2026 17:54:27 +0000 Subject: [PATCH 22/22] librivox.fetch: drop a section title's own leading number Live verification on project 5495 named a file "01 - 1 - Chapter 01 through Chapter 06.mp3": LibriVox titles some sections with their own number. A title that opens with a number and a separator (" - ", ". ", ": ", ") ") loses that prefix when the number is the section's section_number or its 1-based position; a title that is only that number becomes "Section N", and other numbers ("1984") stay. metaops decodes section_number again for it. Part of LibriVox L6-1 (Add from LibriVox). --- PACKAGES.md | 7 +++-- internal/librivoxjob/names.go | 30 +++++++++++++++++++- internal/librivoxjob/names_test.go | 44 ++++++++++++++++++++++++++++++ internal/metaops/librivox.go | 7 +++-- internal/metaops/librivox_test.go | 3 +- 5 files changed, 85 insertions(+), 6 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 6a16563..dafe9f5 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -2638,8 +2638,11 @@ internal/ "LibriVox <id>". Names: control characters and space runs folded, Sanitize, cut to 180 bytes on a rune boundary; files "NN - <section title>.<ext>" (NN the 1-based position, at least two digits, the - count's width; never section_number; an untitled section is - "Section N"; the URL's audio extension, else .mp3). The title's + count's width; never section_number; a title opening with the + section's own number and a separator ("1 - ", "1. ", "1: ", "1) ", + matching its section_number or position) loses it, so 5495's "1 - + Chapter 01 ..." is "01 - Chapter 01 ...", while "1984" stays; an + untitled section, or one that was only its number, is "Section N"; the URL's audio extension, else .mp3). The title's folder taken, once "<Title> (LibriVox <id>)", else exists; a folder already holding this project's librivox.json is the book (rescan only); an author folder that is a listed book or holds audio is diff --git a/internal/librivoxjob/names.go b/internal/librivoxjob/names.go index 6861c8a..6ccdb86 100644 --- a/internal/librivoxjob/names.go +++ b/internal/librivoxjob/names.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "path" + "regexp" "strconv" "strings" "unicode" @@ -127,7 +128,7 @@ func sectionNames(sections []metaops.LibrivoxSection) []string { out := make([]string, len(sections)) for i, s := range sections { n := i + 1 - name := segment(s.Title) + name := segment(dropOwnNumber(s.Title, s.SectionNumber, n)) if name == "" { name = "Section " + strconv.Itoa(n) } @@ -136,6 +137,33 @@ func sectionNames(sections []metaops.LibrivoxSection) []string { return out } +// ownNumber is a section title that opens with a number and a separator +// ("1 - ", "1. ", "1: ", "1) "), or is a number alone (a separator after it +// or not). "1.5 Interlude" is no match: the separator needs a space after it. +var ownNumber = regexp.MustCompile(`^\s*(\d+)(?:\s*[-.:)](?:\s+(.*)|\s*)|\s*)$`) + +// dropOwnNumber is a section's title without the number it opens with when +// that number is the section's own (its section_number, or its 1-based +// position pos): "1 - Chapter 01 through Chapter 06" is section 1's +// "Chapter 01 through Chapter 06", since its file is named "01 - " already. +// A title that is only its own number comes back "" (the caller names it +// "Section N"); any other number ("1984", "1984 - Part One" as section 1) is +// the title's own and stays. +func dropOwnNumber(title, sectionNumber string, pos int) string { + m := ownNumber.FindStringSubmatch(title) + if m == nil { + return title + } + num, err := strconv.Atoi(m[1]) + if err != nil { + return title + } + if sn, err := strconv.Atoi(strings.TrimSpace(sectionNumber)); (err != nil || num != sn) && num != pos { + return title + } + return m[2] +} + // sectionExt is the file's extension from its URL's path (lower case), .mp3 // when it names no audio one. func sectionExt(raw string) string { diff --git a/internal/librivoxjob/names_test.go b/internal/librivoxjob/names_test.go index d33d2c9..2806d07 100644 --- a/internal/librivoxjob/names_test.go +++ b/internal/librivoxjob/names_test.go @@ -166,3 +166,47 @@ func TestAllowedURL(t *testing.T) { t.Error("a nil URL allowed") } } + +// A section title that opens with the section's own number (its +// section_number or its position) loses it: the file name carries it already. +func TestSectionNamesDropOwnNumber(t *testing.T) { + for _, tc := range []struct { + title, number string + pos int + want string + }{ + {"1 - Chapter 01 through Chapter 06", "1", 1, "Chapter 01 through Chapter 06"}, // project 5495 + {"1. Down the Rabbit-Hole", "1", 1, "Down the Rabbit-Hole"}, + {"2: The Pool of Tears", "2", 2, "The Pool of Tears"}, + {"3) A Caucus-Race", "3", 3, "A Caucus-Race"}, + {"03 - Zero padded", "3", 3, "Zero padded"}, + {"0 - Preface", "0", 1, "Preface"}, // section_number from 0 + {"1 - Preface", "0", 1, "Preface"}, // the position + {"7 - Wrong number", "3", 3, "7 - Wrong number"}, // neither: kept + {"1984", "1", 1, "1984"}, // a number title, not a prefix + {"1984 - Part One", "1", 1, "1984 - Part One"}, + {"1.5 Interlude", "1", 1, "1.5 Interlude"}, // no separator before a word + {"12 Angry Men", "12", 12, "12 Angry Men"}, // no separator + {"5", "5", 5, ""}, // only its own number + {"5.", "5", 5, ""}, // nothing after the separator + {"5", "", 5, ""}, + {"Chapter 1", "1", 1, "Chapter 1"}, + {"", "1", 1, ""}, + } { + if got := dropOwnNumber(tc.title, tc.number, tc.pos); got != tc.want { + t.Errorf("dropOwnNumber(%q, %q, %d) = %q, want %q", tc.title, tc.number, tc.pos, got, tc.want) + } + } + got := sectionNames([]metaops.LibrivoxSection{ + {SectionNumber: "1", Title: "1 - Chapter 01 through Chapter 06"}, + {SectionNumber: "2", Title: "2"}, + {SectionNumber: "3", Title: "1984"}, + {SectionNumber: "4", Title: "Introduction"}, + }) + want := []string{"01 - Chapter 01 through Chapter 06.mp3", "02 - Section 2.mp3", "03 - 1984.mp3", "04 - Introduction.mp3"} + for i := range want { + if got[i] != want[i] { + t.Errorf("section %d: %q, want %q", i+1, got[i], want[i]) + } + } +} diff --git a/internal/metaops/librivox.go b/internal/metaops/librivox.go index 63364e2..6eb1c59 100644 --- a/internal/metaops/librivox.go +++ b/internal/metaops/librivox.go @@ -47,8 +47,11 @@ func (p LibrivoxPerson) Name() string { // LibrivoxSection is one section (one file) of a project. type LibrivoxSection struct { - Title string `json:"title"` - ListenURL string `json:"listen_url"` + // SectionNumber is LibriVox's own number for the section (from 0 on some + // projects), a string as LibriVox sends it. + SectionNumber string `json:"section_number"` + Title string `json:"title"` + ListenURL string `json:"listen_url"` } // librivoxDetails is the prefix the feed states url_iarchive with. diff --git a/internal/metaops/librivox_test.go b/internal/metaops/librivox_test.go index a6cad1a..6370686 100644 --- a/internal/metaops/librivox_test.go +++ b/internal/metaops/librivox_test.go @@ -54,7 +54,8 @@ func TestLibrivoxReads(t *testing.T) { t.Fatal(err) } if b.ID != "192" || b.Title != "Johnny Crow's Garden" || len(b.Sections) != 2 || b.CoverartJPG == nil || - b.Sections[1].ListenURL != "https://www.archive.org/download/johnny_crow_garden_librivox/johnnycrow_02_brooke_64kb.mp3" { + b.Sections[1].ListenURL != "https://www.archive.org/download/johnny_crow_garden_librivox/johnnycrow_02_brooke_64kb.mp3" || + b.Sections[1].SectionNumber != "2" { t.Fatalf("%+v", b) } if b.Authors[0].Name() != "L. Leslie Brooke" || b.ArchiveID() != "johnny_crow_garden_librivox" {