diff --git a/README.md b/README.md index 8940f15..5511475 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Files that arrived from elsewhere are read the way their source wrote them, so n Naive string similarity fails on real book titles, so two cases are handled specially: -- **Prefix containment is legitimate.** "Digital Minimalism" vs "Digital Minimalism: Choosing a Focused Life in a Noisy World" is the same book, main title plus subtitle. Scored 0.95. +- **Prefix containment is legitimate, in one direction.** "Digital Minimalism" vs "Digital Minimalism: Choosing a Focused Life in a Noisy World" is the same book, main title plus subtitle. Scored 0.95. A source that stops *short of the filename's main title* is not: "The Dark Tower" answered for a file called "The Dark Tower The Waste Lands" is volume VII, and "Dune" answered for "Dune Messiah" is the first book. The filename says the book is called more than that, so those are capped at 0.69 like any other containment. A source answering exactly the main title before a declared subtitle, or more, keeps 0.95. - **Non-prefix containment is suspicious.** An omnibus titled "The Happiest Baby on the Block and The Happiest Toddler on the Block" contains the title of a book it is not. Capped at 0.69, deliberately below both the threshold that would let it be written and the one that lets an author vouch for a match. A third case is handled separately. **Adaptations and translations are different books that share a title**, so `On Liberty (Squashed Edition)`, `Atomic Habits (Tamil)`, `The Alchemist Graphic Novel` and `Man's Search for Meaning adapted for Young Adults` are capped at 0.55, below even the floor at which a source may contribute a field at all. Every one of those was returned by a live source for the correctly named file. diff --git a/docs/filenames.md b/docs/filenames.md index b21b14e..b6420c9 100644 --- a/docs/filenames.md +++ b/docs/filenames.md @@ -60,7 +60,7 @@ Other observations that shaped the rules: a spaced en dash, a spaced em dash, ` Only when no source identifies the book as first read does `enrich.propose` ask the catalogues about the alternate reading, and it keeps that reading only if a source then identifies the book. A wrong reading cannot score: a source would have to name a book whose title is the author's name and whose author is the title, and two of them would have to agree. The bar for writing is unchanged; the cost is one extra round of queries for a book that was going to be LOW anyway. -A retry with the head of a long title (asking for `Quiet Orchard` when the name says `Quiet Orchard The Year Of Pruning`) was built, measured and removed. It did rescue a name whose colon had been dropped, because Open Library answers nothing for the glued form. It also manufactured a HIGH for the wrong book: a series name glued to a title (`The Dark Tower The Waste Lands`) drew the volume called `The Dark Tower` out of two sources, and a source title that is a strict prefix of the filename title scores 0.95 under the prefix rule, so volume VII's ISBN and series index were proposed for volume III. Anything that makes the tool more willing to write is a change to the safety model, so the retry is gone; a glued subtitle now stays at whatever the full query earns, usually MED, which is honest. The prefix rule itself predates this page and still applies to any source that answers with a strict prefix of the filename title. +A retry with the head of a long title (asking for `Quiet Orchard` when the name says `Quiet Orchard The Year Of Pruning`) was built, measured and removed. It did rescue a name whose colon had been dropped, because Open Library answers nothing for the glued form. It also manufactured a HIGH for the wrong book: a series name glued to a title (`The Dark Tower The Waste Lands`) drew the volume called `The Dark Tower` out of two sources, and a source title that is a strict prefix of the filename title scores 0.95 under the prefix rule, so volume VII's ISBN and series index were proposed for volume III. Anything that makes the tool more willing to write is a change to the safety model, so the retry is gone; a glued subtitle now stays at whatever the full query earns, usually MED, which is honest. The prefix rule has since been made one-directional: a source that stops short of the filename's main title is capped at the containment score (`matching.title_sim`), measured against the same fixtures with no verdict changing, so the glued form can no longer reach HIGH. A name that declares the series name as its main title (`Author - The Dark Tower - The Waste Lands`) still can, because the filename itself says the book is called The Dark Tower and the filename is the ground truth; name the series with its number (`Author - The Dark Tower - 03 - The Waste Lands`) and the series is kept apart from the title. The page paces between books, not inside one, so `web.pause_after` multiplies the wait by the rounds the last book took (`enrich.last_rounds`); Apple's twenty calls a minute hold even when every book is read both ways. In replay mode a fixture set recorded before names had two readings has no key for the second one; that round reports the missing recording and is skipped, while a missing key in the first round stays loud as before. diff --git a/src/ebook_metamend/enrich.py b/src/ebook_metamend/enrich.py index eddbfef..86f1847 100644 --- a/src/ebook_metamend/enrich.py +++ b/src/ebook_metamend/enrich.py @@ -169,7 +169,7 @@ def score(answers: dict[str, dict[str, Any]], facts) -> tuple[list[matching.Sour matching.SourceScore( name=name, title=answer.get('title', ''), - title_score=matching.sim(answer.get('title', ''), facts.title), + title_score=matching.title_sim(answer.get('title', ''), facts.title, facts.query), author_score=matching.best_author_score(answer.get('authors') or [], facts.author), ) for name, answer in answers.items() diff --git a/src/ebook_metamend/matching.py b/src/ebook_metamend/matching.py index 6d6a36c..ee2db6c 100644 --- a/src/ebook_metamend/matching.py +++ b/src/ebook_metamend/matching.py @@ -228,6 +228,34 @@ def sim(a: str | None, b: str | None, *, prefix_bonus: bool = True) -> float: return min(score, ADAPTATION_SCORE) if differently_derived else score +def title_sim(source_title: str | None, filename_title: str, main_title: str) -> float: + """Score a source's title against the filename's, with the direction that + matters for a title. + + ``sim`` treats a prefix as the same book with and without its subtitle in + either direction, which is right when the *source* is the longer one: the + filename says "Sapiens", the catalogue "Sapiens: A Brief History of + Humankind". A source that names *less than the filename's main title* is + another matter. The filename is the ground truth, and it says the book is + called more than that: "The Dark Tower" answered for "The Dark Tower The + Waste Lands" is volume VII, and "The Happiest Baby on the Block" answered + for the two-book omnibus is one of its halves. Both are strict prefixes and + both used to score 0.95, so two catalogues answering with the shorter book + agreed with each other and its ISBN reached HIGH. Capped at CONTAINED_SCORE, + as ``_author_sim`` already caps a name that shortens the filename's. + + ``main_title`` is the filename's title before its declared subtitle + separator (``FilenameFacts.query``): a source answering exactly that, or + more, is the book listed without or with its subtitle and keeps the prefix + score. Only a source that stops short of the main title is under-specified. + """ + score = sim(source_title, filename_title) + answered, main = norm(source_title), norm(main_title) + if answered and main and main.startswith(f'{answered} '): + return min(score, CONTAINED_SCORE) + return score + + #: How a filename names several authors. Commas are deliberately absent: #: "Smith, Jr." is one person, not two. _AUTHOR_SEPARATOR = re.compile(r'\s+(?:and|with|&)\s+', re.I) diff --git a/src/ebook_metamend/sources/apple.py b/src/ebook_metamend/sources/apple.py index 1d30f8c..4e1e477 100644 --- a/src/ebook_metamend/sources/apple.py +++ b/src/ebook_metamend/sources/apple.py @@ -34,7 +34,12 @@ def _html(text: str) -> str: def _best(results: list[dict[str, Any]], title: str, author: str) -> dict[str, Any] | None: - """The result that reads most like the filename: title first, author breaks ties.""" + """The result that reads most like the filename: title first, author breaks ties. + + Scored with the direction the pipeline uses, or a hit that stops short of the + query ("Dune" for "Dune Messiah") ties with the book itself at the prefix + score and wins on result order, only to be capped downstream. + """ ranked = [] for hit in results: if hit.get('kind') != 'ebook': @@ -43,7 +48,7 @@ def _best(results: list[dict[str, Any]], title: str, author: str) -> dict[str, A artist = hit.get('artistName') or '' ranked.append( ( - round(matching.sim(title, name), 2), + round(matching.title_sim(name, title, title), 2), matching.best_author_score([artist], author), hit, ) diff --git a/src/ebook_metamend/sources/inventaire.py b/src/ebook_metamend/sources/inventaire.py index c9ee035..57e7269 100644 --- a/src/ebook_metamend/sources/inventaire.py +++ b/src/ebook_metamend/sources/inventaire.py @@ -78,7 +78,12 @@ def fetch_inventaire(title: str, author: str) -> dict[str, Any] | None: ) hits = (http.get_json(f'{SEARCH_URL}?{query}', timeout=TIMEOUT) or {}).get('results') or [] scored = sorted( - ((round(matching.sim(title, h.get('label') or ''), 2), h) for h in hits if h.get('uri')), + # Same direction as the pipeline: a label that stops short of the query is not the book. + ( + (round(matching.title_sim(h.get('label') or '', title, title), 2), h) + for h in hits + if h.get('uri') + ), key=lambda pair: pair[0], reverse=True, ) diff --git a/tests/test_direct_sources.py b/tests/test_direct_sources.py index 9af2bc7..6606d78 100644 --- a/tests/test_direct_sources.py +++ b/tests/test_direct_sources.py @@ -144,6 +144,18 @@ def test_the_author_breaks_a_title_tie(self): serve({'itunes.apple.com': {'results': [twin, APPLE_HITS['results'][0]]}}) assert fetch_apple('The Quiet Orchard', 'Mara Voss')['authors'] == ['Mara Voss'] + def test_a_hit_that_stops_short_of_the_query_does_not_win_a_tie(self): + """Measured: "Dune" and "Dune Messiah (Dune Chronicles, Book 2)" both + scored 0.95 against "Dune Messiah", and the first book won on result + order; the pipeline then capped it and the book fell out of HIGH.""" + first = dict(APPLE_HITS['results'][0], trackName='The Quiet') + sequel = dict( + APPLE_HITS['results'][0], trackName='The Quiet Orchard (Hill Country, Book 2)' + ) + serve({'itunes.apple.com': {'results': [first, sequel]}}) + record = fetch_apple('The Quiet Orchard', 'Mara Voss') + assert record['title'] == 'The Quiet Orchard (Hill Country, Book 2)' + def test_it_never_claims_a_publisher_or_isbn(self): """Apple has neither, and a blank must not look like a find.""" serve({'itunes.apple.com': APPLE_HITS}) @@ -231,6 +243,24 @@ def test_only_titles_worth_a_round_trip_are_looked_up(self): assert 'wd%3AQ3' not in asked[1] + def test_a_label_that_stops_short_of_the_query_ranks_below_the_book(self): + """Same direction as the pipeline, or the first book of a series ties with + the one asked for and wins the round trip on result order.""" + short = {'uri': 'wd:Q9', 'label': 'The Quiet', 'description': 'novel'} + # Both would score 0.95 as plain prefixes of each other's words. + full = dict(INVENTAIRE_SEARCH['results'][0], label='The Quiet Orchard: A Year of Pruning') + asked = serve_inventaire() + http.set_transport( + lambda url, headers, timeout: ( + json.dumps({'results': [short, full]}).encode() + if 'api/search' in url + else (asked.append(url), json.dumps(INVENTAIRE_WORKS).encode())[1] + ) + ) + fetch_inventaire('The Quiet Orchard', 'Mara Voss') + uris = urllib.parse.parse_qs(urllib.parse.urlsplit(asked[0]).query)['uris'][0] + assert uris.split('|')[0] == full['uri'] + def test_nothing_close_means_no_answer_and_no_second_call(self): asked = serve({'api/search': INVENTAIRE_SEARCH}) assert fetch_inventaire('Completely Different Name', 'Nobody') is None diff --git a/tests/test_matching.py b/tests/test_matching.py index 12171fe..22bc928 100644 --- a/tests/test_matching.py +++ b/tests/test_matching.py @@ -20,6 +20,7 @@ looks_derived, norm, sim, + title_sim, ) @@ -394,6 +395,63 @@ def test_an_exact_author_is_unaffected(self): assert best_author_score(['James Clear'], 'James Clear') == 1.0 +class TestForATitleTheDirectionCarriesTheMeaningToo: + """A source title that extends the filename's is the book with its subtitle. + One that stops short of the filename's main title is under-specified: the + filename says the book is called more than that, and the shorter book is a + different one. Measured: "The Dark Tower" answered for "The Dark Tower The + Waste Lands" is volume VII, and the first volume answered for a two-book + omnibus is half of it; both scored 0.95 and two catalogues agreeing on the + shorter book reached HIGH.""" + + @pytest.mark.parametrize( + 'answered, filename_title, main_title', + [ + ('The Dark Tower', 'The Dark Tower The Waste Lands', 'The Dark Tower The Waste Lands'), + ('Dune', 'Dune Messiah', 'Dune Messiah'), + ( + 'The Happiest Baby on the Block', + 'The Happiest Baby On The Block And The Happiest Toddler On The Block', + 'The Happiest Baby On The Block And The Happiest Toddler On The Block', + ), + # Shorter than the declared main title, not merely shorter than the whole. + ('Digital', 'Digital Minimalism - Choosing a Focused Life', 'Digital Minimalism'), + ], + ) + def test_a_source_that_stops_short_of_the_main_title_cannot_be_strong( + self, answered, filename_title, main_title + ): + assert sim(answered, filename_title) == PREFIX_SCORE, 'sim alone still calls it a prefix' + assert title_sim(answered, filename_title, main_title) == CONTAINED_SCORE + assert title_sim(answered, filename_title, main_title) < TITLE_STRONG + + @pytest.mark.parametrize( + 'answered, filename_title, main_title', + [ + # Exactly the main title: the book listed without its subtitle. + ('Sapiens', 'Sapiens - A Brief History of Humankind', 'Sapiens'), + ('Bad Blood', 'Bad Blood - Secrets And Lies In A Silicon Valley Startup', 'Bad Blood'), + # More than the main title: the book with its subtitle, even cut short. + ('Sapiens: A Brief History', 'Sapiens - A Brief History of Humankind', 'Sapiens'), + # The catalogue is the longer one: the filename left the subtitle off. + ( + 'Digital Minimalism: Choosing a Focused Life', + 'Digital Minimalism', + 'Digital Minimalism', + ), + ], + ) + def test_the_main_title_or_more_still_scores_as_a_prefix( + self, answered, filename_title, main_title + ): + assert title_sim(answered, filename_title, main_title) >= PREFIX_SCORE + + def test_everything_else_is_plain_sim(self): + assert title_sim('Dune', 'Dune', 'Dune') == 1.0 + assert title_sim('Dune', 'Dune Messiah', 'Dune') == PREFIX_SCORE + assert title_sim('', 'Dune Messiah', 'Dune Messiah') == 0.0 + + class TestAFilenameMayNameSeveralAuthors: """Scored against the whole string, each real author looks like a truncation of it and is capped below AUTHOR_STRONG, so every co-authored book in the diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index aec57b9..4ef5eb0 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -10,7 +10,7 @@ from ebook_metamend import calibre, enrich from ebook_metamend.library import Book -from ebook_metamend.matching import SourceScore +from ebook_metamend.matching import CONTAINED_SCORE, SourceScore from ebook_metamend.sources import cache, calibre_plugin, http @@ -204,6 +204,54 @@ def test_a_title_with_no_author_can_never_be_strong(self, tmp_path, monkeypatch) enrich.reset_run_state() +class TestAShorterBookCannotBeWrittenOverALongerOne: + """Two catalogues answering with the earlier volume of a series, whose title + is the head of this one's, used to agree with each other and reach HIGH; the + earlier volume's ISBN and series index were then proposed for this file.""" + + def test_two_sources_naming_the_head_volume_stay_below_high(self, monkeypatch, tmp_path): + head = { + 'title': 'The Quiet Orchard', + 'authors': ['Mara Voss'], + 'isbn': '9781594488849', + 'series': 'The Quiet Orchard', + 'sidx': '1', + } + one = enrich.SOURCES[0].__class__(name='one', fetch=lambda t, a: dict(head), pause=0) + two = enrich.SOURCES[0].__class__(name='two', fetch=lambda t, a: dict(head), pause=0) + monkeypatch.setattr(enrich, 'SOURCES', (one, two)) + monkeypatch.setattr(calibre, 'read_book_metadata', lambda path: {}) + enrich.reset_run_state() + path = tmp_path / 'Mara Voss - The Quiet Orchard The Winter Pruning.epub' + path.write_bytes(b'') + proposal = enrich.propose(Book(stem=path.stem, formats={'.epub': str(path)}), pause=False) + enrich.reset_run_state() + assert proposal is not None + # MED, not HIGH: the author still matches, so the answer keeps a say, + # but the earlier volume's identifiers are behind the --include-low gate. + assert proposal.conf == 'MED' + assert proposal.fn_score == CONTAINED_SCORE + + def test_the_same_head_declared_as_the_main_title_is_the_book_itself( + self, monkeypatch, tmp_path + ): + """ "Mara Voss - The Quiet Orchard - The Winter Pruning" declares "The Quiet + Orchard" as the main title, so a catalogue answering exactly that is the + book listed without its subtitle and HIGH is right.""" + answer = {'title': 'The Quiet Orchard', 'authors': ['Mara Voss']} + one = enrich.SOURCES[0].__class__(name='one', fetch=lambda t, a: dict(answer), pause=0) + two = enrich.SOURCES[0].__class__(name='two', fetch=lambda t, a: dict(answer), pause=0) + monkeypatch.setattr(enrich, 'SOURCES', (one, two)) + monkeypatch.setattr(calibre, 'read_book_metadata', lambda path: {}) + enrich.reset_run_state() + path = tmp_path / 'Mara Voss - The Quiet Orchard - The Winter Pruning.epub' + path.write_bytes(b'') + proposal = enrich.propose(Book(stem=path.stem, formats={'.epub': str(path)}), pause=False) + enrich.reset_run_state() + assert proposal is not None + assert proposal.conf == 'HIGH' + + class TestTheOtherReadingOfANameIsTriedWhenTheFirstFindsNothing: """Half the tools out there write the title first (Calibre, Anna's Archive, Z-Library), half the author first (this tool, Readarr, libgen). The