From 1a516893312c88383be3b4cc3e4a79f4b11ee4bb Mon Sep 17 00:00:00 2001 From: vavallee Date: Sun, 4 Oct 2026 19:51:48 -0300 Subject: [PATCH 1/2] fix(importer): reject a download whose file is in a language the profile does not allow (#2998) The language filter at grab time reads only the release name, so a release that does not say its language passes it. At import #1933 then read the EPUB's dc:language and relabelled the book to match, so an English only library quietly filled with Swedish and Dutch editions that showed as imported and were never searched again. When the author's metadata profile lists its allowed languages (or, with an any language profile, search.preferredLanguage is "en", mirroring the grab time filter) and the downloaded EPUB declares a specific language outside that list, the release is now handled like a disallowed format: import path recorded for Match to book, release blocklisted, download blocked with a message naming both languages, book left Wanted, nothing placed or relabelled. A user locked book language counts as allowed for that book. Unchanged: undeclared, und, mul and unmapped codes; any language profiles (relabelling stays); manual import and Match to book (warn only); library scans and adoption; books already imported. Also folds nb/nob/nn/nno onto nor in NormalizeLanguageCode so Bokmal and Nynorsk files pass a Norwegian profile, and adds LanguageName for readable messages. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- .../2998-reject-disallowed-language.md | 5 + cmd/bindery/main.go | 4 + docs/Troubleshooting-Wiki.md | 8 +- docs/User-Guide-Wiki.md | 35 ++ internal/importer/language_enforcement.go | 181 +++++++++ .../importer/language_enforcement_test.go | 381 ++++++++++++++++++ internal/importer/scanner.go | 59 ++- internal/models/language.go | 59 ++- internal/models/language_test.go | 46 ++- 9 files changed, 763 insertions(+), 15 deletions(-) create mode 100644 changelog.d/2998-reject-disallowed-language.md create mode 100644 internal/importer/language_enforcement.go create mode 100644 internal/importer/language_enforcement_test.go diff --git a/changelog.d/2998-reject-disallowed-language.md b/changelog.d/2998-reject-disallowed-language.md new file mode 100644 index 000000000..76496807a --- /dev/null +++ b/changelog.d/2998-reject-disallowed-language.md @@ -0,0 +1,5 @@ +### Changed +- **A download in a language your profile does not allow is now rejected instead of relabelling the book** (#2998). When a release name did not say its language, a foreign edition could slip past the search filter, and at import Bindery read the EPUB's language and quietly switched the book to it, so an English only library filled up with Swedish and Dutch books that looked imported and were never searched again. Now, when the author's metadata profile lists the languages it allows and the downloaded EPUB declares one outside that list, nothing is imported: the Queue row says which language the file declares and which the profile allows, the release is blocklisted so the next search picks another, and the book stays Wanted. Files that declare no language, or an allowed one in any spelling (en, en-US, eng), import as before, and profiles that allow any language keep the old relabelling. Manual import and Match to book are never refused, and books already in your library are not touched. Thanks to foobarbigtime for the report. + +### Fixed +- **Norwegian Bokmål and Nynorsk count as Norwegian** (#2998). Books and files tagged nb, nob, nn or nno no longer fail a profile that allows Norwegian. diff --git a/cmd/bindery/main.go b/cmd/bindery/main.go index 602287c92..c96492554 100644 --- a/cmd/bindery/main.go +++ b/cmd/bindery/main.go @@ -406,6 +406,10 @@ func main() { // before importing it, and blocklist a release rejected that way so the // next search does not grab it again (#1782). importScanner.WithFormatEnforcement(qualityProfileRepo, blocklistRepo) + // Same for the file's declared language against the author's metadata + // profile: a downloaded EPUB in a language the profile does not allow is + // rejected and blocklisted instead of relabelling the book (#2998). + importScanner.WithLanguageEnforcement(metadataProfileRepo, blocklistRepo) importScanner.WithJobs(bgJobs) // drain manual scans on shutdown (#1458) // Grimmory push pipeline (#826). Config is loaded live per push, so the diff --git a/docs/Troubleshooting-Wiki.md b/docs/Troubleshooting-Wiki.md index 5ca4be208..dbf56e094 100644 --- a/docs/Troubleshooting-Wiki.md +++ b/docs/Troubleshooting-Wiki.md @@ -54,7 +54,7 @@ If your platform can't run a different command, set `BINDERY_DB_FK_CHECK=report` ## A grabbed book never imports -The book shows **grabbed** in History and reaches 100% in your download client, but Bindery never imports it. Usually one of two causes. +The book shows **grabbed** in History and reaches 100% in your download client, but Bindery never imports it. Usually one of the causes below. ### qBittorrent 5.x with an old Bindery @@ -64,6 +64,12 @@ You may see `add torrent failed: {"added_torrent_ids":...}` or `failed to send t **Fix:** upgrade to Bindery **1.12.1 or later** (the current release is recommended). Bindery 1.11.0 and earlier cannot complete torrent grabs against qBittorrent 5.x at all — it is a hard incompatibility. After upgrading, re-grab anything that was stuck. +### The file is in a language your profile does not allow + +The Queue row is **importBlocked** with a message like `file declares Swedish (swe), but the metadata profile "Standard" allows only English (eng)`. The release name did not say its language, so it passed the search filter, but the EPUB inside declares one the author's metadata profile does not allow. Bindery placed nothing, added the release to the Blocklist and left the book Wanted, so the next search picks a different release on its own. See [the file's language is checked at import](User-Guide-Wiki.md#from-wanted-to-your-library) for the full rule. + +If the file is what you want after all, press **Match to book** on the row and pick the book, or import it through **Import → From a folder**; a manual import is never refused for its language. To accept more languages for this author from now on, add them to the metadata profile, or set it to any language. Releases already blocklisted for this stay there until you remove them in **Settings → Blocklist**. + ### Bindery cannot read the completed files **Start with Diagnose.** In **Settings → Download clients**, press **Diagnose** on the client. It works out where Bindery's grabs actually land for ebooks and for audiobooks (the save path Bindery sends, the category's folder, or the client default, and it says which), applies the path remap the same way the importer does, checks that Bindery can read and write the result, and tries a hardlink into each library folder. The panel puts the first thing to fix at the top and shows the paths side by side: where the client puts the files, which remap applied, and where Bindery looks. **Copy report** copies the check results and paths without the host, port or username, so it is safe to paste into an issue or Discord. diff --git a/docs/User-Guide-Wiki.md b/docs/User-Guide-Wiki.md index 041c64aef..51aeaf8a5 100644 --- a/docs/User-Guide-Wiki.md +++ b/docs/User-Guide-Wiki.md @@ -408,6 +408,41 @@ a CWA ingest folder, Grimmory's BookDrop, an Audiobookshelf library scan, webhooks. The three ways of reaching Calibre or CWA are easy to mix up; the [Calibre integration guide](Calibre-Integration-Wiki.md) tells them apart. +**The file's language is checked at import.** The language filter at grab +time can only read the release name, so a release that does not say its +language gets through it. When the download arrives, Bindery reads the +language the EPUB declares (its `dc:language`) and compares it with the +languages the book may be in: the allowed languages on the author's metadata +profile, or, when that profile allows any language, the preferred search +language if it is set to English. That is the same rule the release name was +held to, so a file is never judged more strictly than its name would have +been. + +- **The file is in an allowed language**, declares no language, or declares + one that is not a specific language (`und`, `mul`): it imports as normal. + Region tags are ignored, so `en-US` and `en_GB` count as English, and + Bokmål and Nynorsk (`nb`, `nob`, `nn`, `nno`) count as Norwegian. +- **The file is in a language the profile does not allow**: the release is + treated as the wrong release. Nothing is placed, the Queue row is blocked + with a message naming both languages ("file declares Swedish (swe), but the + metadata profile "Standard" allows only English (eng)"), the release is + added to the Blocklist so the next search picks a different one, and the + book stays Wanted. The failure is in History and is sent to your webhooks + like any other failed import. +- **The profile allows any language** (and the preferred search language is + not English): nothing is rejected. When the file's language differs from + the book's, the book is relabelled to the file's language and History + records a "language corrected" row, because the file on disk is the edition + you actually have. + +To keep a rejected file anyway, use **Match to book** on the Queue row, or +import it through **Import → From a folder**. A manual import is your own +choice and is never refused for its language. If you locked a book's language +by editing it, a file in that language is accepted for that book even when the +profile does not allow it. Only downloads are checked: a library scan, +adopting files already in your library, and books imported before this check +existed are left alone. Audiobooks are not checked yet. + If the library app downstream reads sidecar metadata, turn on **Write a metadata.opf sidecar** in Settings → General (off by default). Bindery then writes a Calibre style `metadata.opf` next to each imported ebook and diff --git a/internal/importer/language_enforcement.go b/internal/importer/language_enforcement.go new file mode 100644 index 000000000..1a7990b6a --- /dev/null +++ b/internal/importer/language_enforcement.go @@ -0,0 +1,181 @@ +package importer + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/vavallee/bindery/internal/importer/formatsniff" + "github.com/vavallee/bindery/internal/models" +) + +// Post-download language enforcement (#2998). +// +// The language filter runs at grab time against the release *name*, and it can +// only act on a name that says its language. A Swedish EPUB in a release named +// "John.Grisham.The.Racketeer.EPUB" passes it. #1933 then read the file's +// dc:language at import and relabelled the book to match, which is right for a +// library that keeps every language and wrong for one restricted to a few: an +// English only library quietly filled with Swedish and Dutch books that showed +// as imported and were never searched again. +// +// So when the book's allowed languages are an explicit list and the file +// declares a language outside it, the release is treated as the wrong release, +// the same way the format check treats a disallowed format: blocked before +// anything is placed, blocklisted so the next search picks another, and the +// book left Wanted. Everything else keeps #1933's relabelling unchanged. + +// manualImportKey marks a context as an import a person asked for (manual +// import, Reassign, the queue's Match to book), as opposed to one the download +// client poll started. +type manualImportKey struct{} + +// withManualImport marks ctx as a manual import. ImportFromPath and +// ImportFilesFromPath are the only entry points that do this, and every caller +// of those is a user action in internal/api. +func withManualImport(ctx context.Context) context.Context { + return context.WithValue(ctx, manualImportKey{}, true) +} + +// isManualImport reports whether withManualImport marked ctx. +func isManualImport(ctx context.Context) bool { + v, _ := ctx.Value(manualImportKey{}).(bool) + return v +} + +// importAllowedLanguages returns the languages a downloaded file for this +// author may be in, and a phrase naming where that list came from. Nil means +// any language. +// +// It mirrors the grab time filter exactly (Scheduler.searchAndGrabFormat and +// IndexerHandler.SearchBook): the author's metadata profile wins, and only +// when that profile allows any language does search.preferredLanguage apply, +// and then only its "en" value, the one indexer.FilterByLanguage acts on. The +// file is held to the same rule its release name was, no stricter. +// +// Every failure reads as "no restriction": refusing a file because a profile +// could not be loaded would block books for a reason the user cannot see. +func (s *Scanner) importAllowedLanguages(ctx context.Context, author *models.Author) ([]string, string) { + if author != nil { + id := models.DefaultMetadataProfileID + if author.MetadataProfileID != nil { + id = *author.MetadataProfileID + } + p, err := s.metadataProfiles.GetByID(ctx, id) + if err != nil { + slog.Warn("import language check: could not load metadata profile, not enforcing", + "profileID", id, "error", err) + } else if p != nil { + if langs := models.ParseAllowedLanguages(p.AllowedLanguages); len(langs) > 0 { + return langs, fmt.Sprintf("the metadata profile %q", p.Name) + } + } + } + if s.settings == nil { + return nil, "" + } + setting, err := s.settings.Get(ctx, "search.preferredLanguage") + if err != nil || setting == nil || setting.Value != "en" { + return nil, "" + } + return []string{"eng"}, "the preferred search language" +} + +// declaredDownloadLanguage returns the language the download's ebook declares: +// the dc:language of the first EPUB that has one, normalised. It considers the +// same files, in the same order, as the import loop's relabelling, so the +// language checked here is the language the book would have been relabelled +// to. Files the quality profile skips are skipped here too. +func (s *Scanner) declaredDownloadLanguage(ctx context.Context, author *models.Author, files []string, slotMediaType string) string { + for _, f := range files { + if !IsEpubFile(f) { + continue + } + if ok, _ := s.allowedFormat(ctx, author, formatsniff.Detect(f), slotMediaType); !ok { + continue + } + meta, err := ReadEpubMetadata(f) + if err != nil || meta.Language == "" { + continue + } + return meta.Language + } + return "" +} + +// definiteLanguage reports whether a normalised code names one specific +// language. A file that says "und" (undetermined), "mul" (several), "zxx" (no +// linguistic content), "mis" (uncoded), a private use code, or something that +// is not an ISO 639-2 code at all ("unknown", an unmapped two letter code) has +// not told us what it is in, and is passed for the same reason an untagged +// release name is: rejecting on ignorance would block legitimate books. +func definiteLanguage(code string) bool { + if len(code) != 3 { + return false + } + for _, r := range code { + if r < 'a' || r > 'z' { + return false + } + } + switch code { + case "und", "mul", "zxx", "mis": + return false + } + // qaa to qtz is reserved for local use. + return code < "qaa" || code > "qtz" +} + +// languageCheck is the outcome of comparing a download's declared language +// with the languages allowed for its book. A zero value means nothing to act +// on. +type languageCheck struct { + disallowed bool + declared string + reason string +} + +// checkDownloadLanguage compares the language the download's EPUB declares +// with the languages its book may be in. +// +// The allowed set is the profile's list (see importAllowedLanguages) plus, +// when the user locked the book's language, that language. A lock is the user +// saying "this book is in X", which outranks the author wide profile for that +// one book, so a Swedish file for a book locked to Swedish is wanted even under +// an English only profile. The lock adds its own language and nothing more. +// +// Unwired (no metadata profile repo) or with no book, it never objects. +func (s *Scanner) checkDownloadLanguage(ctx context.Context, book *models.Book, author *models.Author, files []string, slotMediaType string) languageCheck { + if s.metadataProfiles == nil || book == nil { + return languageCheck{} + } + allowed, source := s.importAllowedLanguages(ctx, author) + if len(allowed) == 0 { + return languageCheck{} + } + declared := s.declaredDownloadLanguage(ctx, author, files, slotMediaType) + if !definiteLanguage(declared) { + return languageCheck{} + } + if models.IsLanguageAllowed(declared, allowed, false) { + return languageCheck{} + } + if book.IsFieldLocked(models.BookFieldLanguage) && book.Language != "" && + models.NormalizeLanguageCode(book.Language) == declared { + return languageCheck{} + } + names := make([]string, 0, len(allowed)) + for _, code := range allowed { + n := models.NormalizeLanguageCode(code) + names = append(names, fmt.Sprintf("%s (%s)", models.LanguageName(n), n)) + } + return languageCheck{ + disallowed: true, + declared: declared, + reason: fmt.Sprintf("file declares %s (%s), but %s allows only %s. "+ + "Not imported, and the release was blocklisted so the next search picks another. "+ + "To keep this file anyway, use Match to book in the Queue or manual import", + models.LanguageName(declared), declared, source, strings.Join(names, ", ")), + } +} diff --git a/internal/importer/language_enforcement_test.go b/internal/importer/language_enforcement_test.go new file mode 100644 index 000000000..a6d736352 --- /dev/null +++ b/internal/importer/language_enforcement_test.go @@ -0,0 +1,381 @@ +package importer + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/vavallee/bindery/internal/db" + "github.com/vavallee/bindery/internal/models" +) + +// langEnforceFixture is a scanner wired for the post-download language check +// (#2998): an author on a metadata profile with the given allowed languages, +// one Wanted ebook, and a grabbed download pointing at a folder the test fills. +type langEnforceFixture struct { + scanner *Scanner + books *db.BookRepo + downloads *db.DownloadRepo + blocklist *db.BlocklistRepo + history *db.HistoryRepo + settings *db.SettingsRepo + notif *spyNotifier + book *models.Book + dl *models.Download + dir string + library string +} + +type langEnforceOpts struct { + allowed string // metadata profile allowed_languages CSV + bookLang string // the catalogue's language for the book + locked bool // the user locked the book's language + preferred string // search.preferredLanguage, "" leaves it unset +} + +func newLangEnforceFixture(t *testing.T, o langEnforceOpts) *langEnforceFixture { + t.Helper() + database, err := db.OpenMemory() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + + ctx := context.Background() + downloads := db.NewDownloadRepo(database) + clients := db.NewDownloadClientRepo(database) + books := db.NewBookRepo(database) + authors := db.NewAuthorRepo(database) + history := db.NewHistoryRepo(database) + settings := db.NewSettingsRepo(database) + profiles := db.NewMetadataProfileRepo(database) + blocklist := db.NewBlocklistRepo(database) + + profile := &models.MetadataProfile{Name: "Restricted", AllowedLanguages: o.allowed, UnknownLanguageBehavior: "pass"} + if err := profiles.Create(ctx, profile); err != nil { + t.Fatalf("create metadata profile: %v", err) + } + author := &models.Author{ + ForeignID: "OL-LANG-A", Name: "John Grisham", SortName: "Grisham, John", + MetadataProvider: "openlibrary", Monitored: true, MetadataProfileID: &profile.ID, + } + if err := authors.Create(ctx, author); err != nil { + t.Fatal(err) + } + book := &models.Book{ + ForeignID: "OL-LANG-B", AuthorID: author.ID, Title: "The Racketeer", + SortTitle: "racketeer", Status: models.BookStatusWanted, Monitored: true, + AnyEditionOK: true, MediaType: models.MediaTypeEbook, MetadataProvider: "openlibrary", + Language: o.bookLang, + } + if o.locked { + book.LockedFields = []string{models.BookFieldLanguage} + } + if err := books.Create(ctx, book); err != nil { + t.Fatal(err) + } + + if err := settings.Set(ctx, "import.mode", "copy"); err != nil { + t.Fatal(err) + } + if o.preferred != "" { + if err := settings.Set(ctx, "search.preferredLanguage", o.preferred); err != nil { + t.Fatal(err) + } + } + + dir := t.TempDir() + library := t.TempDir() + notif := &spyNotifier{} + s := NewScanner(downloads, clients, books, authors, history, library, "", "", "", ""). + WithLanguageEnforcement(profiles, blocklist) + s.WithSettings(settings) + s.WithNotifier(notif) + + dl := &models.Download{ + BookID: &book.ID, GUID: "guid-racketeer", Title: "John.Grisham.The.Racketeer.EPUB", + Status: models.StateCompleted, Protocol: "torrent", + } + if err := downloads.Create(ctx, dl); err != nil { + t.Fatal(err) + } + return &langEnforceFixture{ + scanner: s, books: books, downloads: downloads, blocklist: blocklist, + history: history, settings: settings, notif: notif, book: book, dl: dl, + dir: dir, library: library, + } +} + +// writeEpub drops an EPUB declaring the given dc:language ("" declares none) +// into the download folder. +func (f *langEnforceFixture) writeEpub(t *testing.T, language string) { + t.Helper() + writeEpubWithLanguage(t, filepath.Join(f.dir, "The Racketeer.epub"), language) +} + +// importDownload runs the import the way a download client poll does. +func (f *langEnforceFixture) importDownload(t *testing.T) { + t.Helper() + f.scanner.tryImportInternal(context.Background(), f.dl, f.dir, "qbittorrent", "abc123", "", nil, nil) +} + +func (f *langEnforceFixture) reload(t *testing.T) (*models.Download, *models.Book) { + t.Helper() + ctx := context.Background() + dl, err := f.downloads.GetByID(ctx, f.dl.ID) + if err != nil || dl == nil { + t.Fatalf("reload download: %v", err) + } + book, err := f.books.GetByID(ctx, f.book.ID) + if err != nil || book == nil { + t.Fatalf("reload book: %v", err) + } + return dl, book +} + +func (f *langEnforceFixture) bookFiles(t *testing.T) []models.BookFile { + t.Helper() + files, err := f.books.ListFiles(context.Background(), f.book.ID) + if err != nil { + t.Fatal(err) + } + return files +} + +func (f *langEnforceFixture) blocked(t *testing.T) bool { + t.Helper() + ok, err := f.blocklist.IsBlocked(context.Background(), f.dl.GUID) + if err != nil { + t.Fatal(err) + } + return ok +} + +// assertImported checks the download landed in the library as today. +func (f *langEnforceFixture) assertImported(t *testing.T) *models.Book { + t.Helper() + dl, book := f.reload(t) + if dl.Status != models.StateImported { + t.Fatalf("download status = %q (%s), want %q", dl.Status, dl.ErrorMessage, models.StateImported) + } + if len(f.bookFiles(t)) == 0 { + t.Fatal("no book file recorded for an import that should have landed") + } + if f.blocked(t) { + t.Error("an allowed release was blocklisted") + } + return book +} + +// TestLanguageEnforcement_RejectsDisallowedLanguage is the #2998 report: an +// English only profile, a release whose name does not say its language, and a +// Swedish EPUB inside it. Before the fix the book was relabelled Swedish and +// the file kept; now the release is rejected like any other wrong release. +func TestLanguageEnforcement_RejectsDisallowedLanguage(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "eng"}) + f.writeEpub(t, "sv") + ctx := context.Background() + + f.importDownload(t) + + dl, book := f.reload(t) + if dl.Status != models.StateImportBlocked { + t.Fatalf("download status = %q, want %q: a Swedish file imported against an English only profile", dl.Status, models.StateImportBlocked) + } + for _, want := range []string{"Swedish", "English"} { + if !strings.Contains(dl.ErrorMessage, want) { + t.Errorf("rejection message %q does not name %s", dl.ErrorMessage, want) + } + } + if dl.ImportPath == "" { + t.Error("import path not recorded, so Match to book cannot force the file in") + } + if !f.blocked(t) { + t.Error("release was not blocklisted, so the next search grabs it again") + } + if book.Status != models.BookStatusWanted { + t.Errorf("book status = %q, want %q so the normal search picks another release", book.Status, models.BookStatusWanted) + } + if book.Language != "eng" { + t.Errorf("book language = %q, want %q: a rejected file must not relabel the book", book.Language, "eng") + } + if files := f.bookFiles(t); len(files) != 0 { + t.Errorf("rejected file was recorded on the book: %+v", files) + } + if entries, _ := os.ReadDir(f.library); len(entries) != 0 { + t.Errorf("rejected file was placed in the library: %d entries", len(entries)) + } + if events := correctionEvents(t, ctx, f.history); len(events) != 0 { + t.Errorf("got %d language correction events for a rejected file, want 0", len(events)) + } + failed, err := f.history.ListByType(ctx, models.HistoryEventImportFailed) + if err != nil { + t.Fatal(err) + } + if len(failed) != 1 { + t.Errorf("got %d importFailed history rows, want 1", len(failed)) + } + if f.notif.lookup(notifierEventDownloadFailed) == nil { + t.Error("no downloadFailed notification for the rejection") + } +} + +// TestLanguageEnforcement_AllowedLanguageImports covers region tags and the +// other spellings a dc:language arrives in. Each must compare equal to the +// profile's ISO 639-2/B entry. +func TestLanguageEnforcement_AllowedLanguageImports(t *testing.T) { + cases := []struct{ allowed, declared string }{ + {"eng", "en"}, + {"eng", "en-US"}, + {"eng", "en_GB"}, + {"eng", "EN"}, + {"eng", "eng"}, + {"eng,swe", "sv-SE"}, + {"ger", "deu"}, + {"ger", "de-AT"}, + {"nor", "nob"}, + {"nor", "nb-NO"}, + {"nob", "no"}, + {"nor", "nn"}, + } + for _, c := range cases { + t.Run(c.allowed+"/"+c.declared, func(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: c.allowed}) + f.writeEpub(t, c.declared) + f.importDownload(t) + f.assertImported(t) + }) + } +} + +// TestLanguageEnforcement_UndeclaredOrUnknownImports keeps a file that does not +// say what language it is in. Rejecting on ignorance would block legitimate +// books, the same reason the release name filter passes an untagged name. +func TestLanguageEnforcement_UndeclaredOrUnknownImports(t *testing.T) { + for _, declared := range []string{"", "und", "mul", "zxx", "UND", "unknown", "x-default"} { + t.Run("declared="+declared, func(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "eng"}) + f.writeEpub(t, declared) + f.importDownload(t) + f.assertImported(t) + }) + } +} + +// TestLanguageEnforcement_AnyLanguageKeepsRelabelling: a profile that allows +// every language has nothing to enforce, so #1933's relabelling stays exactly +// as it was for people who keep every language. +func TestLanguageEnforcement_AnyLanguageKeepsRelabelling(t *testing.T) { + for _, allowed := range []string{"any", ""} { + t.Run("allowed="+allowed, func(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: allowed, bookLang: "eng"}) + f.writeEpub(t, "sv") + f.importDownload(t) + book := f.assertImported(t) + if book.Language != "swe" { + t.Errorf("book language = %q, want %q (relabelled from the file)", book.Language, "swe") + } + if events := correctionEvents(t, context.Background(), f.history); len(events) != 1 { + t.Errorf("got %d correction events, want 1", len(events)) + } + }) + } +} + +// TestLanguageEnforcement_PreferredLanguageFallback mirrors the grab time +// filter: with an any language profile, search.preferredLanguage "en" is what +// filtered release names, so it is what the file is held to as well. +func TestLanguageEnforcement_PreferredLanguageFallback(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "any", bookLang: "eng", preferred: "en"}) + f.writeEpub(t, "sv") + f.importDownload(t) + + dl, book := f.reload(t) + if dl.Status != models.StateImportBlocked { + t.Fatalf("download status = %q, want %q", dl.Status, models.StateImportBlocked) + } + if !f.blocked(t) { + t.Error("release was not blocklisted") + } + if book.Language != "eng" { + t.Errorf("book language = %q, want eng", book.Language) + } +} + +// TestLanguageEnforcement_PreferredAnyDoesNotEnforce: "any" (or anything but +// "en", which is all the grab filter acts on) enforces nothing. +func TestLanguageEnforcement_PreferredAnyDoesNotEnforce(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "any", bookLang: "eng", preferred: "any"}) + f.writeEpub(t, "sv") + f.importDownload(t) + f.assertImported(t) +} + +// TestLanguageEnforcement_ManualImportNotRejected: a manual import is the user +// choosing this file, so it is never refused for its language. That includes +// the queue's Match to book, which is how a rejected release is forced in. +func TestLanguageEnforcement_ManualImportNotRejected(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "eng"}) + f.writeEpub(t, "sv") + + f.scanner.ImportFromPath(context.Background(), f.dl, f.dir, "") + + f.assertImported(t) +} + +// TestLanguageEnforcement_ManualMultiFileImportNotRejected covers the other +// manual entry point (#2935). +func TestLanguageEnforcement_ManualMultiFileImportNotRejected(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "eng"}) + f.writeEpub(t, "sv") + + f.scanner.ImportFilesFromPath(context.Background(), f.dl, f.dir, + []string{filepath.Join(f.dir, "The Racketeer.epub")}, models.MediaTypeEbook) + + f.assertImported(t) +} + +// TestLanguageEnforcement_LockedLanguageIsAllowed: a user who locked a book's +// language to Swedish has said this book is Swedish, so a Swedish file for it +// is wanted even when the author's profile is English only. +func TestLanguageEnforcement_LockedLanguageIsAllowed(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "swe", locked: true}) + f.writeEpub(t, "sv") + f.importDownload(t) + book := f.assertImported(t) + if book.Language != "swe" { + t.Errorf("locked language = %q, want swe", book.Language) + } +} + +// TestLanguageEnforcement_LockedLanguageDoesNotWidenToOthers: the lock adds +// its own language and nothing else. A German file for a book locked to +// Swedish under an English only profile is still the wrong release. +func TestLanguageEnforcement_LockedLanguageDoesNotWidenToOthers(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "swe", locked: true}) + f.writeEpub(t, "de") + f.importDownload(t) + + dl, book := f.reload(t) + if dl.Status != models.StateImportBlocked { + t.Fatalf("download status = %q, want %q", dl.Status, models.StateImportBlocked) + } + if book.Language != "swe" { + t.Errorf("locked language = %q, want swe", book.Language) + } +} + +// TestLanguageEnforcement_UnwiredScannerKeepsRelabelling: without the wiring +// the check does not run, so every existing caller behaves as before. +func TestLanguageEnforcement_UnwiredScannerKeepsRelabelling(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "eng"}) + f.scanner.metadataProfiles = nil + f.writeEpub(t, "sv") + f.importDownload(t) + book := f.assertImported(t) + if book.Language != "swe" { + t.Errorf("book language = %q, want swe (relabelled, as before #2998)", book.Language) + } +} diff --git a/internal/importer/scanner.go b/internal/importer/scanner.go index 21ec344b6..a68ddbb13 100644 --- a/internal/importer/scanner.go +++ b/internal/importer/scanner.go @@ -88,8 +88,13 @@ type Scanner struct { // qualityProfiles and blocklist back the post-download format check // (#1782). Both nil disables it entirely, which is what every caller that // has not been wired up gets. - qualityProfiles *db.QualityProfileRepo - blocklist *db.BlocklistRepo + qualityProfiles *db.QualityProfileRepo + blocklist *db.BlocklistRepo + // metadataProfiles backs the post-download language check (#2998): a + // downloaded EPUB declaring a language the author's metadata profile does + // not allow is rejected and blocklisted instead of relabelling the book. + // Nil disables the check, exactly as an unwired format check does. + metadataProfiles *db.MetadataProfileRepo libraryDir string audiobookDir string audiobookDownloadDir string @@ -353,6 +358,15 @@ func (s *Scanner) WithFormatEnforcement(profiles *db.QualityProfileRepo, blockli return s } +// WithLanguageEnforcement wires the repos the post-download language check +// needs (#2998). Without it the check does not run and every import behaves as +// it did before: the file's declared language relabels the book (#1933). +func (s *Scanner) WithLanguageEnforcement(profiles *db.MetadataProfileRepo, blocklist *db.BlocklistRepo) *Scanner { + s.metadataProfiles = profiles + s.blocklist = blocklist + return s +} + // allowedFormat reports whether a file of the given format may fill the given // slot, and the rejection reason when it may not. slotMediaType is the media // type the download is being imported as (detectDownloadFormat's answer, or an @@ -393,14 +407,14 @@ func (s *Scanner) allowedFormat(ctx context.Context, author *models.Author, form decision.Release{Format: format}, models.Book{}) } -// blocklistRejectedRelease records a format-rejected release so the next search -// does not grab the same file again. +// blocklistRejectedRelease records a release rejected for its format (#1782) +// or its language (#2998) so the next search does not grab the same file again. // // Without this the rejection is a loop: the book stays wanted, the next scan // finds the same release, grabs it, downloads it, and rejects it again. The // blocklist is the only thing that makes a rejection stick, and it is also why -// this must stay narrow: it fires on a format the user explicitly disallowed, -// never on a transient import failure. +// this must stay narrow: it fires on a format or language the user explicitly +// disallowed, never on a transient import failure. func (s *Scanner) blocklistRejectedRelease(ctx context.Context, dl *models.Download, reason string) { if s.blocklist == nil || dl == nil || strings.TrimSpace(dl.GUID) == "" { return @@ -413,7 +427,7 @@ func (s *Scanner) blocklistRejectedRelease(ctx context.Context, dl *models.Downl Reason: reason, } if err := s.blocklist.Create(ctx, entry); err != nil { - slog.Warn("could not blocklist a format-rejected release; it may be grabbed again", + slog.Warn("could not blocklist a rejected release; it may be grabbed again", "guid", dl.GUID, "title", dl.Title, "error", err) } } @@ -1396,7 +1410,7 @@ func (s *Scanner) alreadyImportedPath(ctx context.Context, book *models.Book, de // based format detection when non-empty ("ebook" or "audiobook"). func (s *Scanner) ImportFromPath(ctx context.Context, dl *models.Download, path, formatHint string) { defer s.lockManualBook(dl)() - s.tryImportInternal(ctx, dl, path, "", "", formatHint, nil, nil) + s.tryImportInternal(withManualImport(ctx), dl, path, "", "", formatHint, nil, nil) } // ImportFilesFromPath imports several files already on disk as ONE unit: the @@ -1408,7 +1422,7 @@ func (s *Scanner) ImportFromPath(ctx context.Context, dl *models.Download, path, // every file; files are absolute paths. func (s *Scanner) ImportFilesFromPath(ctx context.Context, dl *models.Download, dir string, files []string, formatHint string) { defer s.lockManualBook(dl)() - s.tryImportInternal(ctx, dl, dir, "", "", formatHint, nil, files) + s.tryImportInternal(withManualImport(ctx), dl, dir, "", "", formatHint, nil, files) } // ManualDownloadGUIDPrefix starts the GUID of every synthetic download the @@ -1684,6 +1698,33 @@ func (s *Scanner) tryImportInternal(ctx context.Context, dl *models.Download, do } } + // Post-download language enforcement (#2998). The release name filter + // passes a name that does not say its language, so this is the first + // point where the language is actually known: the EPUB's dc:language. + // When it names a language the book's allowed languages exclude, the + // release is the wrong one, handled exactly like a disallowed format + // above: path recorded for Match to book, release blocklisted, download + // blocked, book left Wanted, nothing placed and nothing relabelled. See + // language_enforcement.go for what counts as allowed. + // + // A manual import is never refused for its language: a person chose this + // file for this book, and the warning in the log is all it gets. The + // relabelling below then records the book's real language, as before. + if detectedFormat != models.MediaTypeAudiobook && len(bookFiles) > 0 { + if lc := s.checkDownloadLanguage(ctx, book, author, bookFiles, detectedFormat); lc.disallowed { + if !isManualImport(ctx) { + slog.Warn("import blocked: file language is not allowed", + "title", dl.Title, "bookID", book.ID, "language", lc.declared) + s.recordUnmatchedImportPath(ctx, dl.ID, downloadPath) + s.blocklistRejectedRelease(ctx, dl, lc.reason) + s.failImport(ctx, dl, models.StateImportBlocked, lc.reason) + return + } + slog.Warn("manual import of a file in a language the profile does not allow; importing it as asked", + "title", dl.Title, "bookID", book.ID, "language", lc.declared) + } + } + // Audiobook path: place the entire download directory as a unit so // multi-part m4b/mp3 files, cover art, and cue sheets stay together. if detectedFormat == models.MediaTypeAudiobook { diff --git a/internal/models/language.go b/internal/models/language.go index 790b6abec..0c15777e8 100644 --- a/internal/models/language.go +++ b/internal/models/language.go @@ -33,10 +33,12 @@ func ParseAllowedLanguages(csv string) []string { return out } -// The three tables below are the only ISO 639 tables in the codebase, and -// NormalizeLanguageCode is the only thing that reads them. There used to be -// four such tables (here, in the indexer's release filter, and twice over in -// the Audible ingestion paths) and they disagreed with each other, so the same +// The four tables below are the only ISO 639 tables in the codebase, and +// NormalizeLanguageCode is the only thing that reads them. (languageNames, +// further down, is a display table keyed by the codes they produce.) There +// used to be four competing copies of these (here, in the indexer's release +// filter, and twice over in the Audible ingestion paths) and they disagreed +// with each other, so the same // language filtered differently depending on which code path happened to be // consulted. Add a language here and every caller learns it at once. @@ -50,6 +52,19 @@ var iso639TwoLetterToB = map[string]string{ "sv": "swe", "no": "nor", "da": "dan", "pl": "pol", "cs": "cze", "tr": "tur", "hi": "hin", "ko": "kor", "ar": "ara", "fi": "fin", "el": "gre", "hu": "hun", "ro": "rum", "ca": "cat", "la": "lat", + // Bokmål and Nynorsk are the two written forms of Norwegian. Folded onto + // the macrolanguage, see iso639IndividualToMacro. + "nb": "nor", "nn": "nor", +} + +// iso639IndividualToMacro folds an ISO 639-3 individual language onto the ISO +// 639-2 macrolanguage Bindery stores, where both are in real use for the same +// books. A Norwegian EPUB or catalogue record is often tagged "nob" (Bokmål) +// or "nno" (Nynorsk) rather than "nor", while the profile editor offers only +// "nor", so without this a profile allowing Norwegian rejected Norwegian +// books and files (#2998). +var iso639IndividualToMacro = map[string]string{ + "nob": "nor", "nno": "nor", } // iso639TermToB maps the ISO 639-2/T (terminology) code onto the 639-2/B @@ -133,9 +148,40 @@ func NormalizeLanguageCode(code string) string { if b, ok := iso639TermToB[code]; ok { return b } + if m, ok := iso639IndividualToMacro[code]; ok { + return m + } return code } +// languageNames gives the English name of each language Bindery recognises, +// keyed by the ISO 639-2/B code NormalizeLanguageCode produces, so a message +// can say "Swedish" rather than "swe". +var languageNames = map[string]string{ + "alb": "Albanian", "ara": "Arabic", "arm": "Armenian", "baq": "Basque", + "bur": "Burmese", "cat": "Catalan", "chi": "Chinese", "cze": "Czech", + "dan": "Danish", "dut": "Dutch", "eng": "English", "fin": "Finnish", + "fre": "French", "geo": "Georgian", "ger": "German", "gre": "Greek", + "hin": "Hindi", "hun": "Hungarian", "ice": "Icelandic", "ita": "Italian", + "jpn": "Japanese", "kor": "Korean", "lat": "Latin", "mac": "Macedonian", + "mao": "Maori", "may": "Malay", "nor": "Norwegian", "per": "Persian", + "pol": "Polish", "por": "Portuguese", "rum": "Romanian", "rus": "Russian", + "slo": "Slovak", "spa": "Spanish", "swe": "Swedish", "tib": "Tibetan", + "tur": "Turkish", "wel": "Welsh", +} + +// LanguageName returns the English name of a language code in any spelling +// NormalizeLanguageCode accepts ("sv", "swe", "sv-SE" all give "Swedish"). A +// code with no known name is returned normalised, so the result is never +// empty for a non-empty input. +func LanguageName(code string) string { + n := NormalizeLanguageCode(code) + if name, ok := languageNames[n]; ok { + return name + } + return n +} + // LanguageCodeVariants returns the provider-facing ISO spellings that are // equivalent to any code in codes. The result includes ISO 639-1, 639-2/B, // and 639-2/T variants where Bindery knows them. Metadata providers that can @@ -163,6 +209,11 @@ func LanguageCodeVariants(codes []string) []string { add(terminology) } } + for individual, macro := range iso639IndividualToMacro { + if macro == normalized { + add(individual) + } + } } variants := make([]string, 0, len(seen)) for code := range seen { diff --git a/internal/models/language_test.go b/internal/models/language_test.go index 9d5208004..47a24b3a1 100644 --- a/internal/models/language_test.go +++ b/internal/models/language_test.go @@ -70,6 +70,16 @@ func TestNormalizeLanguageCode(t *testing.T) { {"english", "eng"}, {"Português", "por"}, {"magyar", "hun"}, + // #2998. Bokmål and Nynorsk fold onto Norwegian, the code profiles + // and the editor use, in every spelling a dc:language carries. + {"nb", "nor"}, + {"nb-NO", "nor"}, + {"nob", "nor"}, + {"NOB", "nor"}, + {"nn", "nor"}, + {"nno", "nor"}, + {"no", "nor"}, + {"nor", "nor"}, // Still not a language, and still round-trips rather than vanishing. {"zulu", "zulu"}, {"not a language", "not a language"}, @@ -199,7 +209,7 @@ func TestNormalizedLanguagesStayInTheEditorVocabulary(t *testing.T) { } produced := map[string]bool{} - for _, table := range []map[string]string{iso639TwoLetterToB, iso639TermToB, iso639NameToB} { + for _, table := range []map[string]string{iso639TwoLetterToB, iso639TermToB, iso639NameToB, iso639IndividualToMacro} { for in, out := range table { // Every table has to agree with the finished canonicaliser, or // one of them is a second opinion again. @@ -237,3 +247,37 @@ func TestNormalizedLanguagesStayInTheEditorVocabulary(t *testing.T) { } } } + +// TestLanguageName covers the names an import rejection uses (#2998): every +// spelling of a code gives one English name, and a code with no name comes +// back normalised rather than empty. +func TestLanguageName(t *testing.T) { + cases := []struct{ in, want string }{ + {"sv", "Swedish"}, + {"swe", "Swedish"}, + {"sv-SE", "Swedish"}, + {"en_GB", "English"}, + {"deu", "German"}, + {"nob", "Norwegian"}, + {"zulu", "zulu"}, + {"XX", "xx"}, + } + for _, tc := range cases { + if got := LanguageName(tc.in); got != tc.want { + t.Errorf("LanguageName(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// TestLanguageNamesCoverEveryNormalisedCode keeps the display table in step +// with the alias tables: a language Bindery can normalise onto must also have +// a name, or a rejection message reads "file declares alb". +func TestLanguageNamesCoverEveryNormalisedCode(t *testing.T) { + for _, table := range []map[string]string{iso639TwoLetterToB, iso639TermToB, iso639NameToB, iso639IndividualToMacro} { + for _, code := range table { + if _, ok := languageNames[code]; !ok { + t.Errorf("no English name for %q", code) + } + } + } +} From 4470c3bd05cd317fe071f5338664e212c4ad003c Mon Sep 17 00:00:00 2001 From: vavallee Date: Sun, 4 Oct 2026 20:17:08 -0300 Subject: [PATCH 2/2] fix(importer): count every declared language and every ISO 639-1 code (#2998) Review of #3000 found two holes in the import language check. Only the first dc:language of the first EPUB counted, so an EPUB tagged fr then en was blocked under an English profile while the same file tagged en then fr imported, and a release with A.epub (sv) and B.epub (en) was rejected while the reversed names imported. EpubMetadata now also carries Languages (every dc:language, normalised, deduped; Language is unchanged for existing callers). The gate classifies each candidate EPUB as allowed when any declared language is allowed, and rejects the release only when some EPUB is disallowed and none is allowed, the shape of the format gate. A mixed release imports the allowed EPUBs and skips the rest in the ebook loop, as the format check skips a format. Relabelling under a restricted profile picks the first allowed declared language, so an "und, en-GB" EPUB no longer relabels the book "und". The two letter table held 25 codes, so uk, he, sk, fa, is and the rest normalised to nothing known and passed as "no language". It now covers all 183 ISO 639-1 codes, with English names for every code it yields, and cmn/yue fold onto chi. The Norwegian fold is renamed iso639NorwegianToB and matches francisrath's #2995 hunk line for line so his rebase drops it cleanly. Docs note that External import mode is not checked. Co-authored-by: francisrath Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016fJcCVbNnKsmj2MAAMwWj9 Signed-off-by: vavallee --- .../2998-reject-disallowed-language.md | 5 +- docs/User-Guide-Wiki.md | 16 +- internal/importer/epubmeta.go | 15 +- internal/importer/epubmeta_fuzz_test.go | 3 +- internal/importer/epubmeta_test.go | 28 +++ internal/importer/language_enforcement.go | 162 ++++++++++++------ .../importer/language_enforcement_test.go | 145 ++++++++++++++++ internal/importer/scanner.go | 44 +++-- internal/models/language.go | 147 ++++++++++++---- internal/models/language_test.go | 89 ++++++++-- 10 files changed, 537 insertions(+), 117 deletions(-) diff --git a/changelog.d/2998-reject-disallowed-language.md b/changelog.d/2998-reject-disallowed-language.md index 76496807a..1bc9190be 100644 --- a/changelog.d/2998-reject-disallowed-language.md +++ b/changelog.d/2998-reject-disallowed-language.md @@ -1,5 +1,6 @@ ### Changed -- **A download in a language your profile does not allow is now rejected instead of relabelling the book** (#2998). When a release name did not say its language, a foreign edition could slip past the search filter, and at import Bindery read the EPUB's language and quietly switched the book to it, so an English only library filled up with Swedish and Dutch books that looked imported and were never searched again. Now, when the author's metadata profile lists the languages it allows and the downloaded EPUB declares one outside that list, nothing is imported: the Queue row says which language the file declares and which the profile allows, the release is blocklisted so the next search picks another, and the book stays Wanted. Files that declare no language, or an allowed one in any spelling (en, en-US, eng), import as before, and profiles that allow any language keep the old relabelling. Manual import and Match to book are never refused, and books already in your library are not touched. Thanks to foobarbigtime for the report. +- **A download in a language your profile does not allow is now rejected instead of relabelling the book** (#2998). When a release name did not say its language, a foreign edition could slip past the search filter, and at import Bindery read the EPUB's language and quietly switched the book to it, so an English only library filled up with Swedish and Dutch books that looked imported and were never searched again. Now, when the author's metadata profile lists the languages it allows and the downloaded EPUB declares one outside that list, nothing is imported: the Queue row says which language the file declares and which the profile allows, the release is blocklisted so the next search picks another, and the book stays Wanted. Files that declare no language, or an allowed one in any spelling (en, en-US, eng), import as before, an EPUB that declares several languages is allowed when any of them is, a release with both an allowed and a disallowed EPUB imports the allowed one, and profiles that allow any language keep the old relabelling. Manual import and Match to book are never refused, and books already in your library are not touched. Thanks to foobarbigtime for the report. ### Fixed -- **Norwegian Bokmål and Nynorsk count as Norwegian** (#2998). Books and files tagged nb, nob, nn or nno no longer fail a profile that allows Norwegian. +- **Norwegian Bokmål and Nynorsk count as Norwegian** (#2998). Books and files tagged nb, nob, nn or nno no longer fail a profile that allows Norwegian. Thanks francisrath. +- **Every two letter language code is recognised** (#2998). Books and files tagged with codes such as uk, he or sk were not recognised as Ukrainian, Hebrew or Slovak, so a Ukrainian EPUB could import under an English only profile. Chinese tagged cmn or yue now counts as Chinese. diff --git a/docs/User-Guide-Wiki.md b/docs/User-Guide-Wiki.md index 51aeaf8a5..35970acfc 100644 --- a/docs/User-Guide-Wiki.md +++ b/docs/User-Guide-Wiki.md @@ -420,9 +420,15 @@ been. - **The file is in an allowed language**, declares no language, or declares one that is not a specific language (`und`, `mul`): it imports as normal. - Region tags are ignored, so `en-US` and `en_GB` count as English, and - Bokmål and Nynorsk (`nb`, `nob`, `nn`, `nno`) count as Norwegian. -- **The file is in a language the profile does not allow**: the release is + Every language an EPUB declares counts, so a bilingual edition tagged + French and English is allowed under an English profile. Region tags are + ignored, so `en-US` and `en_GB` count as English, Bokmål and Nynorsk (`nb`, + `nob`, `nn`, `nno`) count as Norwegian, and `cmn` and `yue` count as + Chinese. Every two letter ISO 639-1 code is recognised. +- **The release holds EPUBs in both allowed and disallowed languages**: the + allowed ones are imported and the others are left out, the same way a + disallowed format inside a release is left out. +- **Every file is in a language the profile does not allow**: the release is treated as the wrong release. Nothing is placed, the Queue row is blocked with a message naming both languages ("file declares Swedish (swe), but the metadata profile "Standard" allows only English (eng)"), the release is @@ -441,7 +447,9 @@ choice and is never refused for its language. If you locked a book's language by editing it, a file in that language is accepted for that book even when the profile does not allow it. Only downloads are checked: a library scan, adopting files already in your library, and books imported before this check -existed are left alone. Audiobooks are not checked yet. +existed are left alone. With the import mode set to **External**, Bindery +hands the download to your other tool without opening its files, so nothing +is checked there either. Audiobooks are not checked yet. If the library app downstream reads sidecar metadata, turn on **Write a metadata.opf sidecar** in Settings → General (off by default). Bindery then diff --git a/internal/importer/epubmeta.go b/internal/importer/epubmeta.go index 27562f9d7..e1d128b26 100644 --- a/internal/importer/epubmeta.go +++ b/internal/importer/epubmeta.go @@ -8,6 +8,7 @@ import ( "io" "path" "path/filepath" + "slices" "strings" "github.com/vavallee/bindery/internal/isbnutil" @@ -24,6 +25,11 @@ type EpubMetadata struct { Author string ISBN string // normalised (digits only); ISBN-13 preferred over ISBN-10 Language string // ISO 639-2/B code from dc:language, normalised ("en" → "eng") + // Languages is every dc:language the OPF declares, normalised, in document + // order with duplicates dropped. Language is its first entry. A bilingual + // edition declares both, and the import language check (#2998) must see + // the second one too. + Languages []string } // IsEpubFile reports whether path is an EPUB we can read embedded metadata from. @@ -206,8 +212,13 @@ func parseOPFMetadata(r io.Reader) (EpubMetadata, error) { // First dc:language wins. Normalise to the ISO 639-2/B code the // language filter and metadata profiles use so a value like "en" // or "en-US" matches an "eng" profile (#1160). - if meta.Language == "" { - meta.Language = models.NormalizeLanguageCode(val) + if code := models.NormalizeLanguageCode(val); code != "" { + if meta.Language == "" { + meta.Language = code + } + if !slices.Contains(meta.Languages, code) { + meta.Languages = append(meta.Languages, code) + } } } case xml.EndElement: diff --git a/internal/importer/epubmeta_fuzz_test.go b/internal/importer/epubmeta_fuzz_test.go index 0eb2da715..b01a0404e 100644 --- a/internal/importer/epubmeta_fuzz_test.go +++ b/internal/importer/epubmeta_fuzz_test.go @@ -5,6 +5,7 @@ import ( "bytes" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -116,7 +117,7 @@ func FuzzReadEpubMetadata(f *testing.F) { if err != nil { // Error path is the fallback-to-filename-parsing contract: it must // hand back a zero value, not partially-populated metadata. - if meta != (EpubMetadata{}) { + if !reflect.DeepEqual(meta, EpubMetadata{}) { t.Fatalf("non-zero metadata %+v alongside error %v", meta, err) } return diff --git a/internal/importer/epubmeta_test.go b/internal/importer/epubmeta_test.go index c04163848..baf98dc37 100644 --- a/internal/importer/epubmeta_test.go +++ b/internal/importer/epubmeta_test.go @@ -4,6 +4,7 @@ import ( "archive/zip" "os" "path/filepath" + "slices" "testing" "github.com/vavallee/bindery/internal/isbnutil" @@ -95,6 +96,33 @@ func TestReadEpubMetadata_LanguageRegionSubtag(t *testing.T) { } } +// TestReadEpubMetadata_AllLanguages: Language stays the first dc:language for +// existing callers, and Languages carries every one, normalised and deduped, +// for the import language check (#2998). +func TestReadEpubMetadata_AllLanguages(t *testing.T) { + opf := ` + + + Le Petit Prince + + fr + en-GB + fr-CA + +` + p := writeTestEpub(t, "content.opf", opf) + meta, err := ReadEpubMetadata(p) + if err != nil { + t.Fatal(err) + } + if meta.Language != "fre" { + t.Errorf("Language = %q, want %q (the first declared)", meta.Language, "fre") + } + if want := []string{"fre", "eng"}; !slices.Equal(meta.Languages, want) { + t.Errorf("Languages = %v, want %v", meta.Languages, want) + } +} + func TestReadEpubMetadata_PrefersAutCreator(t *testing.T) { // First creator is an illustrator; the role="aut" creator must win. opf := ` diff --git a/internal/importer/language_enforcement.go b/internal/importer/language_enforcement.go index 1a7990b6a..2144c0114 100644 --- a/internal/importer/language_enforcement.go +++ b/internal/importer/language_enforcement.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "slices" "strings" "github.com/vavallee/bindery/internal/importer/formatsniff" @@ -82,34 +83,24 @@ func (s *Scanner) importAllowedLanguages(ctx context.Context, author *models.Aut return []string{"eng"}, "the preferred search language" } -// declaredDownloadLanguage returns the language the download's ebook declares: -// the dc:language of the first EPUB that has one, normalised. It considers the -// same files, in the same order, as the import loop's relabelling, so the -// language checked here is the language the book would have been relabelled -// to. Files the quality profile skips are skipped here too. -func (s *Scanner) declaredDownloadLanguage(ctx context.Context, author *models.Author, files []string, slotMediaType string) string { - for _, f := range files { - if !IsEpubFile(f) { - continue - } - if ok, _ := s.allowedFormat(ctx, author, formatsniff.Detect(f), slotMediaType); !ok { - continue +// definiteLanguages returns the codes in langs (already normalised) that name +// one specific language, in order. +func definiteLanguages(langs []string) []string { + var out []string + for _, l := range langs { + if definiteLanguage(l) { + out = append(out, l) } - meta, err := ReadEpubMetadata(f) - if err != nil || meta.Language == "" { - continue - } - return meta.Language } - return "" + return out } // definiteLanguage reports whether a normalised code names one specific // language. A file that says "und" (undetermined), "mul" (several), "zxx" (no // linguistic content), "mis" (uncoded), a private use code, or something that -// is not an ISO 639-2 code at all ("unknown", an unmapped two letter code) has -// not told us what it is in, and is passed for the same reason an untagged -// release name is: rejecting on ignorance would block legitimate books. +// is not an ISO 639-2 code at all ("unknown", "x-default") has not told us +// what it is in, and is passed for the same reason an untagged release name +// is: rejecting on ignorance would block legitimate books. func definiteLanguage(code string) bool { if len(code) != 3 { return false @@ -127,17 +118,59 @@ func definiteLanguage(code string) bool { return code < "qaa" || code > "qtz" } -// languageCheck is the outcome of comparing a download's declared language -// with the languages allowed for its book. A zero value means nothing to act -// on. +// languageCheck is the outcome of comparing the languages a download's EPUBs +// declare with the languages allowed for its book. The zero value means +// nothing is enforced. type languageCheck struct { - disallowed bool - declared string - reason string + // allowed is the effective allowed set (the profile's list plus a locked + // book language). Nil when any language is allowed. + allowed []string + // reject means no EPUB in the download is in an allowed language and at + // least one declares a language that is not: the wrong release. + reject bool + // declared lists the disallowed languages found, for the log. + declared []string + reason string + // skip holds EPUBs in a disallowed language inside a release that also + // carries one in an allowed language. The import loop leaves them out and + // imports the rest, the way it leaves out a disallowed format. + skip map[string]bool +} + +// allows reports whether code is in the effective allowed set. +func (lc languageCheck) allows(code string) bool { + return models.IsLanguageAllowed(code, lc.allowed, false) } -// checkDownloadLanguage compares the language the download's EPUB declares -// with the languages its book may be in. +// relabelLanguage picks the language to record for the book from an imported +// EPUB's declared languages. With a restricted profile it is the first +// declared language the profile allows, so a bilingual "fr, en" edition kept +// under an English only profile does not relabel the book French. Otherwise, +// or when none is allowed (a manual import), it is the first declared one, as +// before #2998. +func (lc languageCheck) relabelLanguage(meta EpubMetadata) string { + if len(lc.allowed) > 0 { + for _, l := range definiteLanguages(meta.Languages) { + if lc.allows(l) { + return l + } + } + } + return meta.Language +} + +// checkDownloadLanguage compares every language every candidate EPUB in the +// download declares with the languages its book may be in. +// +// The candidates are the EPUBs the import loop would place: files the quality +// profile skips are skipped here too. An EPUB counts as allowed when ANY of its +// declared languages is allowed (a bilingual edition, or a stray tag ahead of +// the real one), as disallowed when it declares at least one specific language +// and none is allowed, and as unknown when it declares no specific language. +// The release is rejected only when some EPUB is disallowed and none is +// allowed, the same shape as the format gate, which rejects only when every +// file is disallowed. A release mixing allowed and disallowed EPUBs imports +// the allowed ones and skips the rest. // // The allowed set is the profile's list (see importAllowedLanguages) plus, // when the user locked the book's language, that language. A lock is the user @@ -150,32 +183,63 @@ func (s *Scanner) checkDownloadLanguage(ctx context.Context, book *models.Book, if s.metadataProfiles == nil || book == nil { return languageCheck{} } - allowed, source := s.importAllowedLanguages(ctx, author) - if len(allowed) == 0 { + profileAllowed, source := s.importAllowedLanguages(ctx, author) + if len(profileAllowed) == 0 { return languageCheck{} } - declared := s.declaredDownloadLanguage(ctx, author, files, slotMediaType) - if !definiteLanguage(declared) { - return languageCheck{} + lc := languageCheck{allowed: slices.Clone(profileAllowed)} + if book.IsFieldLocked(models.BookFieldLanguage) && book.Language != "" { + lc.allowed = append(lc.allowed, models.NormalizeLanguageCode(book.Language)) } - if models.IsLanguageAllowed(declared, allowed, false) { - return languageCheck{} + + anyAllowed := false + for _, f := range files { + if !IsEpubFile(f) { + continue + } + if ok, _ := s.allowedFormat(ctx, author, formatsniff.Detect(f), slotMediaType); !ok { + continue + } + meta, err := ReadEpubMetadata(f) + if err != nil { + continue + } + langs := definiteLanguages(meta.Languages) + if len(langs) == 0 { + continue + } + if slices.ContainsFunc(langs, lc.allows) { + anyAllowed = true + continue + } + if lc.skip == nil { + lc.skip = make(map[string]bool) + } + lc.skip[f] = true + for _, l := range langs { + if !slices.Contains(lc.declared, l) { + lc.declared = append(lc.declared, l) + } + } } - if book.IsFieldLocked(models.BookFieldLanguage) && book.Language != "" && - models.NormalizeLanguageCode(book.Language) == declared { - return languageCheck{} + if len(lc.skip) == 0 || anyAllowed { + return lc } - names := make([]string, 0, len(allowed)) - for _, code := range allowed { + + lc.reject = true + lc.reason = fmt.Sprintf("file declares %s, but %s allows only %s. "+ + "Not imported, and the release was blocklisted so the next search picks another. "+ + "To keep this file anyway, use Match to book in the Queue or manual import", + languageList(lc.declared), source, languageList(profileAllowed)) + return lc +} + +// languageList renders codes as "Swedish (swe), German (ger)". +func languageList(codes []string) string { + names := make([]string, 0, len(codes)) + for _, code := range codes { n := models.NormalizeLanguageCode(code) names = append(names, fmt.Sprintf("%s (%s)", models.LanguageName(n), n)) } - return languageCheck{ - disallowed: true, - declared: declared, - reason: fmt.Sprintf("file declares %s (%s), but %s allows only %s. "+ - "Not imported, and the release was blocklisted so the next search picks another. "+ - "To keep this file anyway, use Match to book in the Queue or manual import", - models.LanguageName(declared), declared, source, strings.Join(names, ", ")), - } + return strings.Join(names, ", ") } diff --git a/internal/importer/language_enforcement_test.go b/internal/importer/language_enforcement_test.go index a6d736352..f04ee7492 100644 --- a/internal/importer/language_enforcement_test.go +++ b/internal/importer/language_enforcement_test.go @@ -379,3 +379,148 @@ func TestLanguageEnforcement_UnwiredScannerKeepsRelabelling(t *testing.T) { t.Errorf("book language = %q, want swe (relabelled, as before #2998)", book.Language) } } + +// writeEpubDeclaring writes an EPUB named name into the download folder that +// declares each of langs as its own dc:language, in order. +func (f *langEnforceFixture) writeEpubDeclaring(t *testing.T, name string, langs ...string) string { + t.Helper() + var decl strings.Builder + for _, l := range langs { + decl.WriteString(" " + l + "\n") + } + opf := ` + + + The Racketeer + John Grisham +` + decl.String() + ` +` + src := writeTestEpub(t, "content.opf", opf) + data, err := os.ReadFile(src) + if err != nil { + t.Fatal(err) + } + dst := filepath.Join(f.dir, name) + if err := os.WriteFile(dst, data, 0o644); err != nil { + t.Fatal(err) + } + return dst +} + +// placedLanguages reads back the languages of every EPUB recorded on the book. +func (f *langEnforceFixture) placedLanguages(t *testing.T) [][]string { + t.Helper() + var out [][]string + for _, bf := range f.bookFiles(t) { + meta, err := ReadEpubMetadata(bf.Path) + if err != nil { + t.Fatalf("read placed file %s: %v", bf.Path, err) + } + out = append(out, meta.Languages) + } + return out +} + +// TestLanguageEnforcement_EveryDeclaredLanguageCounts: an EPUB may declare +// several languages (a bilingual edition, or a stray tag ahead of the real +// one). It is allowed when any of them is, whatever order they come in. Only +// the first used to be read, so "fr, en" was blocked while "en, fr" imported. +func TestLanguageEnforcement_EveryDeclaredLanguageCounts(t *testing.T) { + for _, order := range [][]string{{"fr", "en"}, {"en", "fr"}, {"und", "en-GB"}} { + t.Run(strings.Join(order, ","), func(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "eng"}) + f.writeEpubDeclaring(t, "The Racketeer.epub", order...) + f.importDownload(t) + book := f.assertImported(t) + // The relabelling must not pick the disallowed language either. + if book.Language != "eng" { + t.Errorf("book language = %q, want eng (the allowed declared language)", book.Language) + } + }) + } +} + +// TestLanguageEnforcement_MixedReleaseImportsAllowedFile: a release carrying a +// Swedish EPUB and an English one is not the wrong release, it holds the right +// file too. Only the first EPUB used to decide, so "A sv, B en" was rejected +// while "A en, B sv" imported. Now the English file is imported and the +// Swedish one skipped, in either order, the way the format check skips a +// disallowed format inside a mixed release. +func TestLanguageEnforcement_MixedReleaseImportsAllowedFile(t *testing.T) { + for _, swedishFirst := range []bool{true, false} { + name := "english-first" + if swedishFirst { + name = "swedish-first" + } + t.Run(name, func(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "eng"}) + sv, en := "A.epub", "B.epub" + if !swedishFirst { + sv, en = "B.epub", "A.epub" + } + f.writeEpubDeclaring(t, sv, "sv") + f.writeEpubDeclaring(t, en, "en") + f.importDownload(t) + book := f.assertImported(t) + placed := f.placedLanguages(t) + if len(placed) != 1 || len(placed[0]) != 1 || placed[0][0] != "eng" { + t.Errorf("placed files declare %v, want exactly one English file", placed) + } + if book.Language != "eng" { + t.Errorf("book language = %q, want eng", book.Language) + } + }) + } +} + +// TestLanguageEnforcement_AllFilesDisallowedRejects: several EPUBs, none in an +// allowed language, is still the wrong release. +func TestLanguageEnforcement_AllFilesDisallowedRejects(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "eng"}) + f.writeEpubDeclaring(t, "A.epub", "sv") + f.writeEpubDeclaring(t, "B.epub", "de", "und") + f.importDownload(t) + + dl, book := f.reload(t) + if dl.Status != models.StateImportBlocked { + t.Fatalf("download status = %q, want %q", dl.Status, models.StateImportBlocked) + } + for _, want := range []string{"Swedish", "German", "English"} { + if !strings.Contains(dl.ErrorMessage, want) { + t.Errorf("rejection message %q does not name %s", dl.ErrorMessage, want) + } + } + if !f.blocked(t) { + t.Error("release was not blocklisted") + } + if book.Language != "eng" { + t.Errorf("book language = %q, want eng", book.Language) + } +} + +// TestLanguageEnforcement_TwoLetterCodesOutsideTheOldTable: Ukrainian, Hebrew +// and Slovak EPUBs tag themselves "uk", "he", "sk". Those codes were missing +// from the two letter table, so they normalised to nothing Bindery knew, +// counted as "no language declared", imported under an English only profile +// and relabelled the book. +func TestLanguageEnforcement_TwoLetterCodesOutsideTheOldTable(t *testing.T) { + for _, c := range []struct{ code, name string }{ + {"uk", "Ukrainian"}, {"he", "Hebrew"}, {"sk", "Slovak"}, {"fa-IR", "Persian"}, {"is", "Icelandic"}, + } { + t.Run(c.code, func(t *testing.T) { + f := newLangEnforceFixture(t, langEnforceOpts{allowed: "eng", bookLang: "eng"}) + f.writeEpub(t, c.code) + f.importDownload(t) + dl, book := f.reload(t) + if dl.Status != models.StateImportBlocked { + t.Fatalf("download status = %q, want %q: a %s file imported against an English only profile", dl.Status, models.StateImportBlocked, c.name) + } + if !strings.Contains(dl.ErrorMessage, c.name) { + t.Errorf("rejection message %q does not name %s", dl.ErrorMessage, c.name) + } + if book.Language != "eng" { + t.Errorf("book language = %q, want eng (not relabelled)", book.Language) + } + }) + } +} diff --git a/internal/importer/scanner.go b/internal/importer/scanner.go index a68ddbb13..ec7ae8f5c 100644 --- a/internal/importer/scanner.go +++ b/internal/importer/scanner.go @@ -1707,21 +1707,30 @@ func (s *Scanner) tryImportInternal(ctx context.Context, dl *models.Download, do // blocked, book left Wanted, nothing placed and nothing relabelled. See // language_enforcement.go for what counts as allowed. // + // A release that mixes allowed and disallowed EPUBs is not rejected: the + // ebook loop below skips the disallowed ones (langCheck.skip), as it skips + // a disallowed format. + // // A manual import is never refused for its language: a person chose this // file for this book, and the warning in the log is all it gets. The // relabelling below then records the book's real language, as before. + var langCheck languageCheck if detectedFormat != models.MediaTypeAudiobook && len(bookFiles) > 0 { - if lc := s.checkDownloadLanguage(ctx, book, author, bookFiles, detectedFormat); lc.disallowed { - if !isManualImport(ctx) { - slog.Warn("import blocked: file language is not allowed", - "title", dl.Title, "bookID", book.ID, "language", lc.declared) - s.recordUnmatchedImportPath(ctx, dl.ID, downloadPath) - s.blocklistRejectedRelease(ctx, dl, lc.reason) - s.failImport(ctx, dl, models.StateImportBlocked, lc.reason) - return + langCheck = s.checkDownloadLanguage(ctx, book, author, bookFiles, detectedFormat) + if isManualImport(ctx) { + if langCheck.reject { + slog.Warn("manual import of a file in a language the profile does not allow; importing it as asked", + "title", dl.Title, "bookID", book.ID, "languages", langCheck.declared) } - slog.Warn("manual import of a file in a language the profile does not allow; importing it as asked", - "title", dl.Title, "bookID", book.ID, "language", lc.declared) + // A person picked these files; place them all. + langCheck.skip = nil + } else if langCheck.reject { + slog.Warn("import blocked: file language is not allowed", + "title", dl.Title, "bookID", book.ID, "languages", langCheck.declared) + s.recordUnmatchedImportPath(ctx, dl.ID, downloadPath) + s.blocklistRejectedRelease(ctx, dl, langCheck.reason) + s.failImport(ctx, dl, models.StateImportBlocked, langCheck.reason) + return } } @@ -2249,12 +2258,23 @@ func (s *Scanner) tryImportInternal(ctx context.Context, dl *models.Download, do } } + // Skip an EPUB in a language the profile does not allow when the same + // download also carries one in an allowed language (#2998). A download + // where every EPUB is disallowed never reaches this loop: the language + // gate above blocks and blocklists it. + if langCheck.skip[srcFile] { + slog.Info("skipping a file in a language the profile does not allow", + "file", srcFile) + continue + } + // Read the embedded EPUB language while the source is still present // (move mode deletes it on commit). Only when we actually intend to - // backfill, so we never open the zip needlessly. + // backfill, so we never open the zip needlessly. Under a restricted + // profile the first allowed declared language wins (relabelLanguage). if readLanguage && detectedLang == "" && IsEpubFile(srcFile) { if meta, err := ReadEpubMetadata(srcFile); err == nil && meta.Language != "" { - detectedLang = meta.Language + detectedLang = langCheck.relabelLanguage(meta) } } diff --git a/internal/models/language.go b/internal/models/language.go index 0c15777e8..80d32bbaf 100644 --- a/internal/models/language.go +++ b/internal/models/language.go @@ -33,14 +33,14 @@ func ParseAllowedLanguages(csv string) []string { return out } -// The four tables below are the only ISO 639 tables in the codebase, and -// NormalizeLanguageCode is the only thing that reads them. (languageNames, -// further down, is a display table keyed by the codes they produce.) There -// used to be four competing copies of these (here, in the indexer's release -// filter, and twice over in the Audible ingestion paths) and they disagreed -// with each other, so the same -// language filtered differently depending on which code path happened to be -// consulted. Add a language here and every caller learns it at once. +// The alias tables below are the only ISO 639 tables in the codebase, and +// NormalizeLanguageCode is the only thing that reads them (languageNames, +// further down, is a display table keyed by the codes they produce). There +// used to be four competing copies (here, in the indexer's release filter, +// and twice over in the Audible ingestion paths) and they disagreed with each +// other, so the same language filtered differently depending on which code +// path happened to be consulted. Add a language here and every caller learns +// it at once. // iso639TwoLetterToB maps ISO 639-1 two-letter codes to the ISO 639-2/B // three-letter vocabulary Bindery stores in Book.Language and metadata-profile @@ -49,22 +49,43 @@ func ParseAllowedLanguages(csv string) []string { var iso639TwoLetterToB = map[string]string{ "en": "eng", "fr": "fre", "de": "ger", "nl": "dut", "es": "spa", "it": "ita", "pt": "por", "ja": "jpn", "zh": "chi", "ru": "rus", - "sv": "swe", "no": "nor", "da": "dan", "pl": "pol", "cs": "cze", + "sv": "swe", "no": "nor", "nb": "nor", "nn": "nor", "da": "dan", "pl": "pol", "cs": "cze", "tr": "tur", "hi": "hin", "ko": "kor", "ar": "ara", "fi": "fin", "el": "gre", "hu": "hun", "ro": "rum", "ca": "cat", "la": "lat", - // Bokmål and Nynorsk are the two written forms of Norwegian. Folded onto - // the macrolanguage, see iso639IndividualToMacro. - "nb": "nor", "nn": "nor", -} - -// iso639IndividualToMacro folds an ISO 639-3 individual language onto the ISO -// 639-2 macrolanguage Bindery stores, where both are in real use for the same -// books. A Norwegian EPUB or catalogue record is often tagged "nob" (Bokmål) -// or "nno" (Nynorsk) rather than "nor", while the profile editor offers only -// "nor", so without this a profile allowing Norwegian rejected Norwegian -// books and files (#2998). -var iso639IndividualToMacro = map[string]string{ - "nob": "nor", "nno": "nor", + // The rest of ISO 639-1, so a file tagged "uk", "he" or "sk" is read as + // the language it names rather than as no language at all (#2998). + "aa": "aar", "ab": "abk", "ae": "ave", "af": "afr", "ak": "aka", + "am": "amh", "an": "arg", "as": "asm", "av": "ava", "ay": "aym", + "az": "aze", "ba": "bak", "be": "bel", "bg": "bul", "bi": "bis", + "bm": "bam", "bn": "ben", "bo": "tib", "br": "bre", "bs": "bos", + "ce": "che", "ch": "cha", "co": "cos", "cr": "cre", "cu": "chu", + "cv": "chv", "cy": "wel", "dv": "div", "dz": "dzo", "ee": "ewe", + "eo": "epo", "et": "est", "eu": "baq", "fa": "per", "ff": "ful", + "fj": "fij", "fo": "fao", "fy": "fry", "ga": "gle", "gd": "gla", + "gl": "glg", "gn": "grn", "gu": "guj", "gv": "glv", "ha": "hau", + "he": "heb", "ho": "hmo", "hr": "hrv", "ht": "hat", "hy": "arm", + "hz": "her", "ia": "ina", "id": "ind", "ie": "ile", "ig": "ibo", + "ii": "iii", "ik": "ipk", "io": "ido", "is": "ice", "iu": "iku", + "jv": "jav", "ka": "geo", "kg": "kon", "ki": "kik", "kj": "kua", + "kk": "kaz", "kl": "kal", "km": "khm", "kn": "kan", "kr": "kau", + "ks": "kas", "ku": "kur", "kv": "kom", "kw": "cor", "ky": "kir", + "lb": "ltz", "lg": "lug", "li": "lim", "ln": "lin", "lo": "lao", + "lt": "lit", "lu": "lub", "lv": "lav", "mg": "mlg", "mh": "mah", + "mi": "mao", "mk": "mac", "ml": "mal", "mn": "mon", "mr": "mar", + "ms": "may", "mt": "mlt", "my": "bur", "na": "nau", "nd": "nde", + "ne": "nep", "ng": "ndo", "nr": "nbl", "nv": "nav", "ny": "nya", + "oc": "oci", "oj": "oji", "om": "orm", "or": "ori", "os": "oss", + "pa": "pan", "pi": "pli", "ps": "pus", "qu": "que", "rm": "roh", + "rn": "run", "rw": "kin", "sa": "san", "sc": "srd", "sd": "snd", + "se": "sme", "sg": "sag", "si": "sin", "sk": "slo", "sl": "slv", + "sm": "smo", "sn": "sna", "so": "som", "sq": "alb", "sr": "srp", + "ss": "ssw", "st": "sot", "su": "sun", "sw": "swa", "ta": "tam", + "te": "tel", "tg": "tgk", "th": "tha", "ti": "tir", "tk": "tuk", + "tl": "tgl", "tn": "tsn", "to": "ton", "ts": "tso", "tt": "tat", + "tw": "twi", "ty": "tah", "ug": "uig", "uk": "ukr", "ur": "urd", + "uz": "uzb", "ve": "ven", "vi": "vie", "vo": "vol", "wa": "wln", + "wo": "wol", "xh": "xho", "yi": "yid", "yo": "yor", "za": "zha", + "zu": "zul", } // iso639TermToB maps the ISO 639-2/T (terminology) code onto the 639-2/B @@ -80,6 +101,13 @@ var iso639TermToB = map[string]string{ "msa": "may", "nld": "dut", "ron": "rum", "slk": "slo", "zho": "chi", } +// iso639NorwegianToB folds the two Norwegian written standards onto the +// macrolanguage code. Bokmål and Nynorsk have their own ISO 639-2 codes, and +// Norwegian catalogues (EPUB dc:language, the National Library) use them, but +// a metadata profile offers only "Norwegian" and stores "nor". Without this a +// profile allowing Norwegian rejected every book tagged "nob" or "nno". +var iso639NorwegianToB = map[string]string{"nob": "nor", "nno": "nor"} + // iso639NameToB maps a language written out as a word onto its 639-2/B code. // Audible reports languages this way ("english", "german"), and release names // carry them in English and in the language's own spelling. @@ -117,6 +145,12 @@ var iso639NameToB = map[string]string{ "latin": "lat", } +// iso639ChineseToB folds the ISO 639-3 codes for the two Chinese languages +// books are written in onto the macrolanguage code a profile stores, the way +// iso639NorwegianToB does for Norwegian. EPUBs tag themselves "cmn" or "yue" +// as well as "zh". +var iso639ChineseToB = map[string]string{"cmn": "chi", "yue": "chi"} + // NormalizeLanguageCode canonicalizes a language code from any source (an // EPUB's dc:language, provider metadata, a hand-edited metadata profile) into // the lowercased ISO 639-2/B form the language filter compares against. It @@ -139,6 +173,9 @@ func NormalizeLanguageCode(code string) string { if i := strings.IndexAny(code, "-_"); i > 0 { code = code[:i] } + if b, ok := iso639ChineseToB[code]; ok { + return b + } if len(code) == 2 { if b, ok := iso639TwoLetterToB[code]; ok { return b @@ -148,8 +185,8 @@ func NormalizeLanguageCode(code string) string { if b, ok := iso639TermToB[code]; ok { return b } - if m, ok := iso639IndividualToMacro[code]; ok { - return m + if b, ok := iso639NorwegianToB[code]; ok { + return b } return code } @@ -158,16 +195,54 @@ func NormalizeLanguageCode(code string) string { // keyed by the ISO 639-2/B code NormalizeLanguageCode produces, so a message // can say "Swedish" rather than "swe". var languageNames = map[string]string{ - "alb": "Albanian", "ara": "Arabic", "arm": "Armenian", "baq": "Basque", - "bur": "Burmese", "cat": "Catalan", "chi": "Chinese", "cze": "Czech", - "dan": "Danish", "dut": "Dutch", "eng": "English", "fin": "Finnish", - "fre": "French", "geo": "Georgian", "ger": "German", "gre": "Greek", - "hin": "Hindi", "hun": "Hungarian", "ice": "Icelandic", "ita": "Italian", - "jpn": "Japanese", "kor": "Korean", "lat": "Latin", "mac": "Macedonian", - "mao": "Maori", "may": "Malay", "nor": "Norwegian", "per": "Persian", - "pol": "Polish", "por": "Portuguese", "rum": "Romanian", "rus": "Russian", - "slo": "Slovak", "spa": "Spanish", "swe": "Swedish", "tib": "Tibetan", - "tur": "Turkish", "wel": "Welsh", + "aar": "Afar", "abk": "Abkhazian", "afr": "Afrikaans", "aka": "Akan", + "alb": "Albanian", "amh": "Amharic", "ara": "Arabic", "arg": "Aragonese", + "arm": "Armenian", "asm": "Assamese", "ava": "Avaric", "ave": "Avestan", + "aym": "Aymara", "aze": "Azerbaijani", "bak": "Bashkir", "bam": "Bambara", + "baq": "Basque", "bel": "Belarusian", "ben": "Bengali", "bis": "Bislama", + "bos": "Bosnian", "bre": "Breton", "bul": "Bulgarian", "bur": "Burmese", + "cat": "Catalan", "cha": "Chamorro", "che": "Chechen", "chi": "Chinese", + "chu": "Church Slavic", "chv": "Chuvash", "cor": "Cornish", + "cos": "Corsican", "cre": "Cree", "cze": "Czech", "dan": "Danish", + "div": "Divehi", "dut": "Dutch", "dzo": "Dzongkha", "eng": "English", + "epo": "Esperanto", "est": "Estonian", "ewe": "Ewe", "fao": "Faroese", + "fij": "Fijian", "fin": "Finnish", "fre": "French", + "fry": "Western Frisian", "ful": "Fulah", "geo": "Georgian", + "ger": "German", "gla": "Scottish Gaelic", "gle": "Irish", + "glg": "Galician", "glv": "Manx", "gre": "Greek", "grn": "Guarani", + "guj": "Gujarati", "hat": "Haitian Creole", "hau": "Hausa", "heb": "Hebrew", + "her": "Herero", "hin": "Hindi", "hmo": "Hiri Motu", "hrv": "Croatian", + "hun": "Hungarian", "ibo": "Igbo", "ice": "Icelandic", "ido": "Ido", + "iii": "Sichuan Yi", "iku": "Inuktitut", "ile": "Interlingue", + "ina": "Interlingua", "ind": "Indonesian", "ipk": "Inupiaq", + "ita": "Italian", "jav": "Javanese", "jpn": "Japanese", + "kal": "Kalaallisut", "kan": "Kannada", "kas": "Kashmiri", "kau": "Kanuri", + "kaz": "Kazakh", "khm": "Khmer", "kik": "Kikuyu", "kin": "Kinyarwanda", + "kir": "Kyrgyz", "kom": "Komi", "kon": "Kongo", "kor": "Korean", + "kua": "Kuanyama", "kur": "Kurdish", "lao": "Lao", "lat": "Latin", + "lav": "Latvian", "lim": "Limburgish", "lin": "Lingala", + "lit": "Lithuanian", "ltz": "Luxembourgish", "lub": "Luba-Katanga", + "lug": "Ganda", "mac": "Macedonian", "mah": "Marshallese", + "mal": "Malayalam", "mao": "Maori", "mar": "Marathi", "may": "Malay", + "mlg": "Malagasy", "mlt": "Maltese", "mon": "Mongolian", "nau": "Nauru", + "nav": "Navajo", "nbl": "South Ndebele", "nde": "North Ndebele", + "ndo": "Ndonga", "nep": "Nepali", "nor": "Norwegian", "nya": "Chichewa", + "oci": "Occitan", "oji": "Ojibwa", "ori": "Odia", "orm": "Oromo", + "oss": "Ossetian", "pan": "Punjabi", "per": "Persian", "pli": "Pali", + "pol": "Polish", "por": "Portuguese", "pus": "Pashto", "que": "Quechua", + "roh": "Romansh", "rum": "Romanian", "run": "Rundi", "rus": "Russian", + "sag": "Sango", "san": "Sanskrit", "sin": "Sinhala", "slo": "Slovak", + "slv": "Slovenian", "sme": "Northern Sami", "smo": "Samoan", "sna": "Shona", + "snd": "Sindhi", "som": "Somali", "sot": "Southern Sotho", "spa": "Spanish", + "srd": "Sardinian", "srp": "Serbian", "ssw": "Swati", "sun": "Sundanese", + "swa": "Swahili", "swe": "Swedish", "tah": "Tahitian", "tam": "Tamil", + "tat": "Tatar", "tel": "Telugu", "tgk": "Tajik", "tgl": "Tagalog", + "tha": "Thai", "tib": "Tibetan", "tir": "Tigrinya", "ton": "Tongan", + "tsn": "Tswana", "tso": "Tsonga", "tuk": "Turkmen", "tur": "Turkish", + "twi": "Twi", "uig": "Uyghur", "ukr": "Ukrainian", "urd": "Urdu", + "uzb": "Uzbek", "ven": "Venda", "vie": "Vietnamese", "vol": "Volapuk", + "wel": "Welsh", "wln": "Walloon", "wol": "Wolof", "xho": "Xhosa", + "yid": "Yiddish", "yor": "Yoruba", "zha": "Zhuang", "zul": "Zulu", } // LanguageName returns the English name of a language code in any spelling @@ -209,9 +284,9 @@ func LanguageCodeVariants(codes []string) []string { add(terminology) } } - for individual, macro := range iso639IndividualToMacro { + for standard, macro := range iso639NorwegianToB { if macro == normalized { - add(individual) + add(standard) } } } diff --git a/internal/models/language_test.go b/internal/models/language_test.go index 47a24b3a1..faa66e258 100644 --- a/internal/models/language_test.go +++ b/internal/models/language_test.go @@ -5,6 +5,7 @@ import ( "reflect" "regexp" "slices" + "strings" "testing" ) @@ -63,6 +64,11 @@ func TestNormalizeLanguageCode(t *testing.T) { {"ron", "rum"}, {"ell", "gre"}, {"deu-DE", "ger"}, + // Bokmål and Nynorsk fold onto the "nor" a profile stores. + {"nob", "nor"}, + {"nno", "nor"}, + {"nb", "nor"}, + {"nn-NO", "nor"}, // A language written out as a word, which is how Audible and Audnex // report it and how release names carry it. {"German", "ger"}, @@ -70,16 +76,19 @@ func TestNormalizeLanguageCode(t *testing.T) { {"english", "eng"}, {"Português", "por"}, {"magyar", "hun"}, - // #2998. Bokmål and Nynorsk fold onto Norwegian, the code profiles - // and the editor use, in every spelling a dc:language carries. - {"nb", "nor"}, - {"nb-NO", "nor"}, - {"nob", "nor"}, + // #2998. The rest of ISO 639-1, which EPUBs use, and the Chinese + // individual languages. + {"uk", "ukr"}, + {"he", "heb"}, + {"sk", "slo"}, + {"fa-IR", "per"}, + {"cy", "wel"}, + {"is", "ice"}, + {"sr-Latn", "srp"}, + {"cmn", "chi"}, + {"yue-HK", "chi"}, {"NOB", "nor"}, - {"nn", "nor"}, - {"nno", "nor"}, - {"no", "nor"}, - {"nor", "nor"}, + {"nb-NO", "nor"}, // Still not a language, and still round-trips rather than vanishing. {"zulu", "zulu"}, {"not a language", "not a language"}, @@ -206,10 +215,28 @@ func TestNormalizedLanguagesStayInTheEditorVocabulary(t *testing.T) { // side is the standard's own spelling of the same language. "alb", "arm", "baq", "bur", "geo", "ice", "mac", "mao", "may", "per", "slo", "tib", "wel", + // The rest of ISO 639-1 (#2998). An EPUB tagged "uk" has to read as + // Ukrainian, not as no language, or the import language check lets + // it through an English only profile. + "aar", "abk", "afr", "aka", "amh", "arg", "asm", "ava", "ave", "aym", + "aze", "bak", "bam", "bel", "ben", "bis", "bos", "bre", "bul", "cha", + "che", "chu", "chv", "cor", "cos", "cre", "div", "dzo", "epo", "est", + "ewe", "fao", "fij", "fry", "ful", "gla", "gle", "glg", "glv", "grn", + "guj", "hat", "hau", "heb", "her", "hmo", "hrv", "ibo", "ido", "iii", + "iku", "ile", "ina", "ind", "ipk", "jav", "kal", "kan", "kas", "kau", + "kaz", "khm", "kik", "kin", "kir", "kom", "kon", "kua", "kur", "lao", + "lav", "lim", "lin", "lit", "ltz", "lub", "lug", "mah", "mal", "mar", + "mlg", "mlt", "mon", "nau", "nav", "nbl", "nde", "ndo", "nep", "nya", + "oci", "oji", "ori", "orm", "oss", "pan", "pli", "pus", "que", "roh", + "run", "sag", "san", "sin", "slv", "sme", "smo", "sna", "snd", "som", + "sot", "srd", "srp", "ssw", "sun", "swa", "tah", "tam", "tat", "tel", + "tgk", "tgl", "tha", "tir", "ton", "tsn", "tso", "tuk", "twi", "uig", + "ukr", "urd", "uzb", "ven", "vie", "vol", "wln", "wol", "xho", "yid", + "yor", "zha", "zul", } produced := map[string]bool{} - for _, table := range []map[string]string{iso639TwoLetterToB, iso639TermToB, iso639NameToB, iso639IndividualToMacro} { + for _, table := range languageAliasTables() { for in, out := range table { // Every table has to agree with the finished canonicaliser, or // one of them is a second opinion again. @@ -259,6 +286,8 @@ func TestLanguageName(t *testing.T) { {"en_GB", "English"}, {"deu", "German"}, {"nob", "Norwegian"}, + {"uk", "Ukrainian"}, + {"cmn", "Chinese"}, {"zulu", "zulu"}, {"XX", "xx"}, } @@ -273,7 +302,7 @@ func TestLanguageName(t *testing.T) { // with the alias tables: a language Bindery can normalise onto must also have // a name, or a rejection message reads "file declares alb". func TestLanguageNamesCoverEveryNormalisedCode(t *testing.T) { - for _, table := range []map[string]string{iso639TwoLetterToB, iso639TermToB, iso639NameToB, iso639IndividualToMacro} { + for _, table := range languageAliasTables() { for _, code := range table { if _, ok := languageNames[code]; !ok { t.Errorf("no English name for %q", code) @@ -281,3 +310,41 @@ func TestLanguageNamesCoverEveryNormalisedCode(t *testing.T) { } } } + +// languageAliasTables is every table NormalizeLanguageCode reads. +func languageAliasTables() []map[string]string { + return []map[string]string{iso639TwoLetterToB, iso639TermToB, iso639NameToB, iso639NorwegianToB, iso639ChineseToB} +} + +// TestTwoLetterTableCoversISO6391 pins the two letter table to the whole of +// ISO 639-1 (#2998). The table used to hold 25 codes, so an EPUB tagged "uk", +// "he" or "sk" normalised to a code nothing recognised and was treated as +// declaring no language at all. +func TestTwoLetterTableCoversISO6391(t *testing.T) { + // The 183 current ISO 639-1 codes ("bh" was withdrawn in 2021). + iso6391 := strings.Fields(` + aa ab ae af ak am an ar as av ay az ba be bg bi bm bn bo br bs ca ce ch + co cr cs cu cv cy da de dv dz ee el en eo es et eu fa ff fi fj fo fr fy + ga gd gl gn gu gv ha he hi ho hr ht hu hy hz ia id ie ig ii ik io is it + iu ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln lo + lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl nn no nr nv ny + oc oj om or os pa pi pl ps pt qu rm rn ro ru rw sa sc sd se sg si sk sl + sm sn so sq sr ss st su sv sw ta te tg th ti tk tl tn to tr ts tt tw ty + ug uk ur uz ve vi vo wa wo xh yi yo za zh zu`) + if len(iso6391) != 183 { + t.Fatalf("fixture lists %d codes, want 183", len(iso6391)) + } + for _, code := range iso6391 { + b, ok := iso639TwoLetterToB[code] + if !ok { + t.Errorf("ISO 639-1 %q is missing from iso639TwoLetterToB", code) + continue + } + if len(b) != 3 { + t.Errorf("iso639TwoLetterToB[%q] = %q, want a three letter code", code, b) + } + } + if len(iso639TwoLetterToB) != len(iso6391) { + t.Errorf("iso639TwoLetterToB has %d entries, want %d", len(iso639TwoLetterToB), len(iso6391)) + } +}