diff --git a/README.he.md b/README.he.md index 280d06b..5cdc4f3 100644 --- a/README.he.md +++ b/README.he.md @@ -204,6 +204,31 @@ magiah book --book "C:\ספר חדש.txt" --out results # קובץ מהמ הבאה מול הקורפוס. בתיקייה `real_word/` נשמר קובץ אחד לכל ספר שנסרק כך, והיא אינה מתנקה מעצמה; מחיקתה רק מחייבת סריקה חוזרת עם `--book-verify-ctx`. +`--book-domain-max SHARE` (כבוי כברירת מחדל; 0.05 הוא הערך שנמדד) הוא **שער +תחום** לספר כולו. המילון מכיר את הלשון שבה כתובה רוב הספרייה; ספר בעברית +מודרנית, בשפה אחרת או בשיטת כתיב אחרת מלא מילים שהמילון לא ראה, וכללי הנדירות +מסמנים בו מילים נכונות. לפני הדיווח הסריקה סופרת את מילות הספר בנות 3 אותיות +ומעלה (בלי ראשי תיבות) שהמילון מכיר לכל היותר `rare_max` פעמים מחוץ לספר +עצמו. כשיותר מ-SHARE מהן אינן מוכרות (ובספר יש לפחות `book_domain_min_words`, +200, מופעים של מילים כאלה, ולפחות `book_domain_min_types`, 100, מילים שונות — טקסט קצר החוזר על מילים מעטות אינו נמדד), הממצאים שנשענים על נדירות המילה מושמטים — `edit1_*`, +`spelling_variant`, `missing_space`, `lost_quotes` ו-`real_word` — והסריקה +אומרת זאת ביומן ובפריט `domain` של התוצאה (השיעור, וכמה ממצאים מכל סוג +הושמטו). ממצאי אותיות סופיות, פרופיל ה-OCR של הספר ורווחים מיותרים נשארים. הדילוג על שורה +זרה (`foreign_ratio`) ממשיך לפעול שורה-שורה בכל ספר; השער הוא ההחלטה ברמת +הספר שמעליו. במילון של ספריית אוצריא 99% מספרי ההלכה, התלמוד, השו"ת והמשנה +מתחת ל-2%; ה-0.6% מספרי הספרייה שמעל 5% הם בעיקר תרגומים ארמיים, ועוד תרגומים +בערבית-יהודית ובסורית וכמה קובצי פיוט ומדרש; ספרים דיגיטליים בעברית מודרנית +מגיעים ל-6–16%. + +השער מסרב לפעול, אומר זאת ביומן ומשאיר את כל הממצאים (`domain.refused` הוא +`small` או `partial`, ו-`out` שקר) כשהמילון אינו יכול לשמש קנה מידה: הוא מכיל +פחות מ-`book_domain_min_lexicon` (1,000,000) מילים בסך הכול, או שנבנה חלקית +עם `--allow-unread`. מילון כמעט ריק אינו מכיר שום מילה, ולכן כל ספר רגיל היה +נראה זר. בספר שנקרא מהמאגר (`db`/`library`) מופעיו שלו מופחתים תחילה מספירת +המילון, כבכלל של חזרת הספר ובדיקת המילים האמיתיות; מקובץ (`file`) לא מופחת +דבר. כשהשער פועל, גם חלופות הפיצול של ממצאי `missing_space` שהושמטו מושמטות, +ו-`real_word.hints` סופר את מה שעוד מוצג. + > דורש `lexicon.pkl` קיים — כלומר הרצה אחת של `magiah lexicon` (או `all`). **בדיקת תיקון בדיעבד** (`magiah guard`): diff --git a/README.md b/README.md index 791a60e..bc1d348 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,39 @@ judged until the next corpus-scope scan. `real_word/` holds one file per book scanned this way and is never pruned; deleting it only means the next book-scope scan needs a `--book-verify-ctx` rescan first. +`--book-domain-max SHARE` (off by default; 0.05 is a measured value) is a +**domain gate** for the whole book. The lexicon knows the language most of +the library is written in; a book in modern Hebrew, another language or another spelling +system is full of words it has never seen, and the rarity rules then flag +correct words. Before reporting, the scan counts the book's words of 3+ +letters (abbreviations aside) that the lexicon has at most `rare_max` times +apart from the book itself. When more than SHARE of them are unknown (and +the book has at least `book_domain_min_words`, 200, such word occurrences +**and** at least `book_domain_min_types`, 100, different ones: a short text +that repeats a few words is not judged), the +findings that rest on a word being rare are left out — `edit1_*`, +`spelling_variant`, `missing_space`, `lost_quotes` and `real_word` — and +the scan says so in its log and in the result's `domain` entry (the share +and how many findings of each kind were left out). Final-letter findings, +the book's OCR profile and extra spaces stay. The foreign-line skip +(`foreign_ratio`) still works line by line inside any book; the gate is +the book-level decision on top of it. On the Otzaria library lexicon, +99% of the books of halacha, Talmud, responsa and Mishnah measure under 2%; +the 0.6% of library books above 5% are mostly Aramaic targums, plus the +Judeo-Arabic and Syriac translations and a few piyyut and midrash collections; +digital modern-Hebrew books measure 6-16%. + +The gate refuses to apply, says so in the log and keeps every finding +(`domain.refused` is `small` or `partial`, `out` is false) when the lexicon +cannot serve as a yardstick: it counted fewer than `book_domain_min_lexicon` +(1,000,000) words in all, or it was built partial under `--allow-unread`. +A near-empty lexicon knows no word, so every ordinary book would look +foreign. A book read from the corpus (`db`/`library`) has its own +occurrences taken away from the lexicon's count first, the same rule as the +book-repeat and real-word checks; a `file` has none taken away. When the gate +acts, split alternatives of left-out `missing_space` findings are left out +too, and `real_word.hints` counts what is still shown. + > Requires an existing `lexicon.pkl` — i.e. one prior `magiah lexicon` (or > `all`) run. diff --git a/magiah/book_scan.py b/magiah/book_scan.py index 6513a00..cdf3e49 100644 --- a/magiah/book_scan.py +++ b/magiah/book_scan.py @@ -62,7 +62,7 @@ from .book_source import _rel_within, load_book from .config import Config from .corpus import OTZARIA_DB -from .normalize import tokenize +from .normalize import is_abbrev, tokenize from .textsource import ReadStats LOCAL_TYPES = core.LOCAL_TYPES @@ -139,6 +139,87 @@ def detect_in_book(book, freq, cfg, out_dir, progress=None): return lex, errors, split_cands, present, lsource +# Findings the domain gate (Config.book_domain_max) leaves out: the rules +# that call a word wrong because the lexicon finds it rare and knows a +# similar, a split or an abbreviated common form. In a book outside the +# lexicon's domain most rare words are simply words the lexicon never saw. +# Final-letter rules, the book's reviewed OCR profile and extra spaces do +# not rest on the word being rare, and stay. +DOMAIN_TYPES = ('edit1_sub', 'edit1_ins', 'edit1_del', 'edit1_swap', + 'spelling_variant', 'missing_space', 'lost_quotes', + realword.ERRTYPE) + + +def domain_on(cfg): + """Is the domain gate on (``Config.book_domain_max`` above 0)?""" + try: + return float(cfg.book_domain_max or 0) > 0 + except (TypeError, ValueError): + return False + + +def book_domain(book, freq, present, cfg): + """How much of `book` the lexicon does not know, and whether the book is + outside the lexicon's domain. + + Counted over the book's words of 3+ letters that are not abbreviations; + a word is unknown when the lexicon has it at most ``rare_max`` times + apart from this book (a file from outside the corpus was never counted + in it, so nothing is taken away). Returns a dict: ``words`` (their + occurrences), ``types`` (how many different ones), ``unknown`` + (occurrences), ``share``, ``max`` and ``out`` (True when the share is + above ``book_domain_max``; never for a book with fewer than + ``book_domain_min_words`` such occurrences or fewer than + ``book_domain_min_types`` different words: the minimum is both, so a + short text that repeats a few words is not judged). + + Own count: a book read from the corpus (kind 'db' or 'library') is part + of what the lexicon counted, once, tokenized as ``_book_tokens`` does + (the lexicon's counting reads text the same way), so its own + occurrences are taken away before asking whether the lexicon knows the + word. This is the same assumption as the book-repeat rule + (``freq <= present`` is "only this book", ``core.book_repeat_value``) + and the real-word check (``own = present`` unless kind is 'file'). A + 'file' is never taken away from, even one that happens to sit inside + the corpus library: that can only understate the unknown share (the + safe side for a gate that drops findings). + """ + own = book.kind != 'file' + words = types = unknown = 0 + for w, c in present.items(): + if len(w) < 3 or is_abbrev(w): + continue + words += c + types += 1 + if freq.get(w, 0) - (c if own else 0) <= cfg.rare_max: + unknown += c + share = unknown / words if words else 0.0 + limit = float(cfg.book_domain_max) + return {'words': words, 'types': types, 'unknown': unknown, + 'share': round(share, 4), 'max': limit, + 'out': (words >= cfg.book_domain_min_words + and types >= cfg.book_domain_min_types + and share > limit)} + + +def domain_lexicon_problem(freq, cfg, out_dir): + """Why the lexicon cannot tell what is outside its domain, or None. + + The share of unknown words is only meaningful against a lexicon that + knows a language. Two cases make nearly every word of a normal book + look unknown, so the gate would drop the findings of an ordinary book: + a lexicon with too few words in all (``book_domain_min_lexicon``; + 'small'), and one built partial under ``--allow-unread`` + ('partial': a missing part of the library is missing words). Returns + ``'small'`` / ``'partial'``; the gate then does not apply. + """ + if core.accepted_gaps(out_dir, 'lexicon'): + return 'partial' + if sum(freq.values()) < cfg.book_domain_min_lexicon: + return 'small' + return None + + def _resolve_splits(errors, split_cands, freq, lex, seq_counts, scope): """Record each word's best-evidenced split; return all alternatives.""" alts_out, idx = {}, 0 @@ -471,16 +552,47 @@ def scan_loaded_book(book, freq, cfg, out_dir, verify_ctx=False, spec=None, _covered(joins, ocr)) rows.extend(real_word.pop('rows')) + domain = None + dropped_words = set() # words whose every finding the gate left out + if domain_on(cfg): + domain = book_domain(book, freq, present, cfg) + refused = domain_lexicon_problem(freq, cfg, out_dir) + if refused: # measured, but not enough of a yardstick to act on + domain['out'] = False + domain['refused'] = refused + domain['lexicon_words'] = sum(freq.values()) + domain['lexicon_min'] = cfg.book_domain_min_lexicon + left_out = Counter() + if domain['out']: + kept = [] + for r in rows: + if r['errtype'] in DOMAIN_TYPES: + left_out[r['errtype']] += 1 + dropped_words.add(r['word']) + else: + kept.append(r) + rows = kept + dropped_words -= {r['word'] for r in rows} + if real_word is not None: # its summary counts what is shown + n_rw = left_out.get(realword.ERRTYPE, 0) + real_word['hints'] -= n_rw + real_word['domain_left_out'] = n_rw + domain['left_out'] = dict(left_out) + say(book_domain_message(book.title, domain)) + space_rows = [{ 'part1': p1, 'part2': p2, 'joined': j, 'join_freq': jf, 'origin': book.origin, 'source': book.title, 'ref': ref, 'unit': unit, 'doc': book.doc, 'snippet': snip, } for unit, ref, p1, p2, j, jf, snip in joins] + # a split of a word whose finding the gate left out is not shown either, + # and no word is primary for a missing_space finding that is gone + gone = 'missing_space' in left_out_types(domain) split_alts = [{'word': w, 'parts': p, 'observed': o, 'expected': e, 'chosen': bool(ch), - 'is_primary': errors[w][1] == 'missing_space'} - for w, lst in alts.items() + 'is_primary': errors[w][1] == 'missing_space' and not gone} + for w, lst in alts.items() if w not in dropped_words for p, o, e, ch in lst] # same rule as a full scan: accepted is not complete @@ -518,9 +630,40 @@ def scan_loaded_book(book, freq, cfg, out_dir, verify_ctx=False, spec=None, } if real_word is not None: # off: the result is as it always was out['real_word'] = real_word + if domain is not None: # likewise + out['domain'] = domain return out +def left_out_types(domain): + """The finding types the domain gate left out (empty: it did not act).""" + return set((domain or {}).get('left_out') or ()) + + +def book_domain_message(title, domain): + """The Hebrew line that reports the domain gate's verdict on a book.""" + share, limit = domain['share'], domain['max'] + if domain.get('refused'): + if domain['refused'] == 'partial': + why = 'המילון נבנה חלקית (--allow-unread)' + else: + why = (f'המילון קטן מדי: {domain["lexicon_words"]:,} מילים, ' + f'נדרשים לפחות {domain["lexicon_min"]:,} (book_domain_min_lexicon)') + return (f'[book] שער התחום (book_domain_max) לא הופעל: {why}; ' + f'כל הממצאים נשמרים (השיעור המדוד: {share:.1%})') + if not domain['out']: + why = (f'מעט מדי מילים למדידה ({domain["words"]:,} מופעים, {domain["types"]:,} שונות)' + if share > limit else 'בתוך תחום המילון') + return (f'[book] מילים שהמילון אינו מכיר: {share:.1%} ' + f'(סף {limit:.0%}) — {why}') + n = sum(domain['left_out'].values()) + return (f'[book] «{title}» מחוץ לתחום המילון: {share:.1%} ממילותיו ' + f'(3 אותיות ומעלה) כמעט אינן במאגר, יותר מ-{limit:.0%} ' + f'(book_domain_max). {n:,} ממצאים שנשענים על נדירות המילה ' + f'הושמטו; ממצאי אותיות סופיות, פרופיל OCR ורווחים מיותרים ' + f'נשארו') + + # --------------------------------------------------------------------------- # optional: real-word errors judged by the neighbouring words # --------------------------------------------------------------------------- diff --git a/magiah/cli.py b/magiah/cli.py index ad3501f..43f08c7 100644 --- a/magiah/cli.py +++ b/magiah/cli.py @@ -122,6 +122,8 @@ def _run_book_cmd(args, spec, cfg, out_dir): source = args.book_source or _guess_book_source(args.book) if args.book_real_word: cfg.real_word = 1 + if args.book_domain_max is not None: + cfg.book_domain_max = args.book_domain_max from .webui import db as uidb # recorded apart from the pipeline's runs: a book scan never touches the # full scan's results, so it must neither raise nor clear their warning @@ -208,6 +210,13 @@ def main(argv=None): 'another existing word, judged by the neighbouring ' 'words; the corpus counts are made by a ' '--book-verify-ctx scan and kept for later scans') + bk.add_argument('--book-domain-max', type=float, default=None, + metavar='SHARE', + help='leave out the rarity-based findings of a book in ' + 'which more than this share of the words (e.g. ' + '0.05) is unknown to the lexicon: modern Hebrew, ' + 'another language or spelling (default 0: off)') + gd = ap.add_argument_group( 'guard: check a corrected text against the lexicon (changes nothing)') gd.add_argument('--before', metavar='FILE', diff --git a/magiah/config.py b/magiah/config.py index 79d6652..928d094 100644 --- a/magiah/config.py +++ b/magiah/config.py @@ -62,6 +62,24 @@ class Config: real_word_rep: int = 3 # ... a word the book uses this often (>= 1), # and more often than the suggestion, is its # own spelling + book_domain_max: float = 0.0 # book scan: a book with more than this + # share of its words (3+ letters, not + # abbreviations) rare in the lexicon apart from + # the book itself (<= rare_max) is outside the + # lexicon's domain (modern Hebrew, another + # language or spelling): its frequency-based + # findings (book_scan.DOMAIN_TYPES) are left out + # the result says so. 0 (or less): off + book_domain_min_words: int = 200 # ... judged only on books with at + # least this many such word OCCURRENCES ... + book_domain_min_types: int = 100 # ... and this many DIFFERENT such + # words (a short repetitive text reaches the + # first without being a sample of a language) + book_domain_min_lexicon: int = 1000000 # the gate applies only when the + # lexicon counted at least this many words in + # all (a near-empty lexicon knows no word: every + # book would look foreign) and was not built + # partial (--allow-unread); 0: no size floor rw_corpus: int = 0 # corpus scan (1 = on): weigh each edit-1 / # spelling-variant correction against the # original between the same neighbours diff --git a/magiah/webui/hebrew.py b/magiah/webui/hebrew.py index 4e9fc20..34b2e89 100644 --- a/magiah/webui/hebrew.py +++ b/magiah/webui/hebrew.py @@ -624,6 +624,41 @@ def status_hebrew(key): 'מילה שחוזרת בספר לפחות כך וכך פעמים, ויותר מההצעה, היא ' 'הכתיב של הספר ואינה מסומנת לפי ההקשר. לפחות 1. ברירת מחדל: 3.', }, + 'book_domain_max': { + 'hebrew': 'ספר מחוץ לתחום המילון (book_domain_max)', + 'explanation': + 'בסריקת ספר בודד בלבד: ספר שחלק גדול כזה ממילותיו (3 אותיות ' + 'ומעלה) כמעט אינו במאגר — למשל עברית מודרנית או שפה אחרת — ' + 'רוב הסימונים בו יהיו מילים נכונות. לכן הממצאים שנשענים על ' + 'נדירות המילה (אות שגויה, כתיב מלא/חסר, רווח חסר, מירכאות ' + 'חסרות, מילים קיימות לפי ההקשר) מושמטים, והסריקה אומרת זאת ' + 'במפורש. אותיות סופיות, פרופיל OCR ורווחים מיותרים נשארים. ' + 'ערך מומלץ: 0.05. ברירת מחדל: 0 (כבוי).', + }, + 'book_domain_min_words': { + 'hebrew': 'מינימום מילים לתחום (book_domain_min_words)', + 'explanation': + 'בספר קצר מדי התחום אינו נמדד: כאן מספר המופעים של מילים בנות 3 אותיות ' + 'ומעלה (לא ראשי תיבות) שהשיעור נמדד עליהן. נדרש גם מספר מילים שונות ' + '(book_domain_min_types), כדי שטקסט קצר החוזר על מילים מעטות לא ייחשב. ' + 'ברירת מחדל: 200.', + }, + 'book_domain_min_types': { + 'hebrew': 'מינימום מילים שונות לתחום (book_domain_min_types)', + 'explanation': + 'כמה מילים שונות (3 אותיות ומעלה, לא ראשי תיבות) צריכות להיות בספר ' + 'כדי שתחומו יימדד; טקסט קצר החוזר על מילים מעטות אינו דגימה של לשון. ' + 'ברירת מחדל: 100.', + }, + 'book_domain_min_lexicon': { + 'hebrew': 'גודל מינימלי של המילון לשער התחום (book_domain_min_lexicon)', + 'explanation': + 'השער פועל רק מול מילון שמכיל לפחות כמות זו של מילים (מופעים). מילון ' + 'קטן מדי, או שנבנה חלקית (עם --allow-unread), אינו מכיר את הלשון, ' + 'וספר רגיל ייראה מולו זר; אז השער אינו מופעל, כל הממצאים נשמרים ' + 'והסריקה אומרת זאת ביומן. ברירת מחדל: 1,000,000; 0: בלי רף גודל ' + '(מילון חלקי עדיין פוסל את השער).', + }, 'workers': { 'hebrew': 'מספר תהליכים מקבילים (workers)', 'explanation': diff --git a/tests/test_book_domain.py b/tests/test_book_domain.py new file mode 100644 index 0000000..757d452 --- /dev/null +++ b/tests/test_book_domain.py @@ -0,0 +1,329 @@ +# -*- coding: utf-8 -*- +"""Book domain gate (Config.book_domain_max): a book most of whose words the +lexicon has never seen loses its rarity-based findings, and the scan says +so; structural / OCR-profile / extra-space findings stay. Off by default, +and off means the result is exactly what it was.""" +import os +import shutil +import tempfile +import unittest +from collections import Counter +from types import SimpleNamespace +from unittest import mock + +from magiah import book_scan +from magiah.config import Config +from magiah.webui import hebrew, scanner + +from test_detection import A_LINES, B_LINES, BOOK_A, LIB_TOP, PROFILES, \ + run_full, small_cfg, write_lib, write_profiles + +LETTERS = 'זטצקפגש' + + +def made_up(n): + """`n` different invented 4-letter words, none of them in the library.""" + out = [] + for a in LETTERS: + for b in LETTERS: + for c in LETTERS: + out.append('ק' + a + b + c) + return out[:n] + + +class Measure(unittest.TestCase): + + def cfg(self, **kw): + return Config(**{'book_domain_max': 0.10, + 'book_domain_min_words': 4, + 'book_domain_min_types': 1, **kw}) + + def test_share_counts_long_words_only(self): + present = Counter({'ספר': 6, 'קזזז': 2, 'אב': 50, 'רמב"ם': 50}) + d = book_scan.book_domain(SimpleNamespace(kind='file'), + {'ספר': 100}, present, self.cfg()) + self.assertEqual((d['words'], d['unknown']), (8, 2)) + self.assertEqual(d['share'], 0.25) + self.assertTrue(d['out']) + + def test_own_occurrences_do_not_count_for_a_corpus_book(self): + # the lexicon counted this book: its 3 uses of the word are its own + freq = {'ספר': 100, 'קזזז': 3} + present = Counter({'ספר': 9, 'קזזז': 3}) + lib = book_scan.book_domain(SimpleNamespace(kind='library'), freq, + present, self.cfg()) + self.assertEqual(lib['unknown'], 3) + # a file from outside the corpus: the lexicon's count is all others' + f = book_scan.book_domain(SimpleNamespace(kind='file'), freq, + present, self.cfg()) + self.assertEqual(f['unknown'], 0) + + def test_threshold_is_strict(self): + present = Counter({'ספר': 9, 'קזזז': 1}) + d = book_scan.book_domain(SimpleNamespace(kind='file'), + {'ספר': 100}, present, self.cfg()) + self.assertEqual(d['share'], 0.1) + self.assertFalse(d['out']) + + def test_short_book_is_not_judged(self): + present = Counter({'קזזז': 3}) + d = book_scan.book_domain(SimpleNamespace(kind='file'), {}, present, + self.cfg()) + self.assertEqual(d['share'], 1.0) + self.assertFalse(d['out']) + self.assertIn('מעט מדי מילים', + book_scan.book_domain_message('x', dict(d, left_out={}))) + + def test_switch(self): + self.assertFalse(book_scan.domain_on(Config())) + self.assertFalse(book_scan.domain_on(Config(book_domain_max=-1))) + self.assertFalse(book_scan.domain_on(Config(book_domain_max=None))) + self.assertFalse(book_scan.domain_on( + Config(book_domain_max=float('nan')))) + self.assertTrue(book_scan.domain_on(Config(book_domain_max=0.1))) + + def test_labels(self): + for key in ('book_domain_max', 'book_domain_min_words', + 'book_domain_min_types', 'book_domain_min_lexicon'): + self.assertIn(key, hebrew.CONFIG_LABELS) + + def test_minimum_is_distinct_words_too(self): + # 500 occurrences of ONE unknown word: a short repetitive text, not + # a sample of a language; the default minimums (200 occurrences, + # 100 different words) do not judge it + self.assertEqual((Config().book_domain_min_words, + Config().book_domain_min_types), (200, 100)) + present = Counter({'אאאא': 500}) + f = SimpleNamespace(kind='file') + d = book_scan.book_domain(f, {}, present, self.cfg( + book_domain_min_words=200, book_domain_min_types=100)) + self.assertEqual((d['words'], d['types'], d['share']), (500, 1, 1.0)) + self.assertFalse(d['out']) + # ... while 120 different unknown words, twice each, are judged + many = Counter({w: 2 for w in made_up(120)}) + d = book_scan.book_domain(f, {}, many, self.cfg( + book_domain_min_words=200, book_domain_min_types=100)) + self.assertEqual((d['words'], d['types']), (240, 120)) + self.assertTrue(d['out']) + # both minimums apply: enough different words, too few occurrences + d = book_scan.book_domain(f, {}, Counter({w: 1 for w in made_up(120)}), + self.cfg(book_domain_min_words=200, + book_domain_min_types=100)) + self.assertFalse(d['out']) + + def test_own_count_is_the_only_this_book_rule(self): + # for a corpus book a word is unknown (rare_max 0) exactly when the + # lexicon has it no more often than the book does: the same + # `freq <= present` test as book_repeat's only_this_book + for kind in ('library', 'db'): + for freq_w, n in ((5, 5), (6, 5), (4, 5), (1, 1), (2, 1)): + d = book_scan.book_domain( + SimpleNamespace(kind=kind), {'אבגד': freq_w}, + Counter({'אבגד': n}), self.cfg(rare_max=0)) + self.assertEqual(d['unknown'] == n, freq_w <= n, + (kind, freq_w, n)) + + def test_lexicon_problem(self): + freq = {'אבג': 10} + none = mock.patch.object(book_scan.core, 'accepted_gaps', + return_value={}) + with none: + self.assertEqual(book_scan.domain_lexicon_problem( + freq, Config(), 'x'), 'small') + self.assertEqual(book_scan.domain_lexicon_problem( + freq, Config(book_domain_min_lexicon=11), 'x'), 'small') + self.assertIsNone(book_scan.domain_lexicon_problem( + freq, Config(book_domain_min_lexicon=10), 'x')) + self.assertIsNone(book_scan.domain_lexicon_problem( + {}, Config(book_domain_min_lexicon=0), 'x')) + # built partial under --allow-unread: refused whatever its size + part = mock.patch.object(book_scan.core, 'accepted_gaps', + return_value={'lexicon': {'unread_rows': 1}}) + with part: + self.assertEqual(book_scan.domain_lexicon_problem( + freq, Config(book_domain_min_lexicon=0), 'x'), 'partial') + + def test_hebrew_strings_are_not_mojibake(self): + # the bytes are UTF-8 Hebrew: no C1 controls, no geresh-mojibake + bad = [chr(c) for c in range(0x80, 0xa0)] + ['\u05f3', '\u05b2', + '\u20ac'] + import magiah.book_scan as bs + import magiah.config as cf + for mod in (bs, cf, hebrew): + with open(mod.__file__, 'rb') as f: + text = f.read().decode('utf-8') + self.assertFalse([c for c in bad if c in text], mod.__file__) + for key in ('book_domain_max', 'book_domain_min_words', + 'book_domain_min_types', 'book_domain_min_lexicon'): + lab = hebrew.CONFIG_LABELS[key] + for s in (lab['hebrew'], lab['explanation']): + self.assertTrue(any('א' <= c <= 'ת' for c in s)) + self.assertFalse([c for c in bad if c in s]) + + +class Scan(unittest.TestCase): + """A small library; outside .txt files are scanned against it.""" + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.mkdtemp(prefix='magiah_domain_') + cls.lib = os.path.join(cls.tmp, 'lib') + cls.out = os.path.join(cls.tmp, 'out') + os.makedirs(cls.out) + write_lib(cls.lib, {BOOK_A: A_LINES, f'{LIB_TOP}/b.txt': B_LINES}) + cls.spec = {'type': 'library', 'path': cls.lib} + write_profiles(cls.out, PROFILES) + run_full(cls.spec, small_cfg(), cls.out) + # the fixture's slips amid ordinary text: in the lexicon's domain + cls.inside = os.path.join(cls.tmp, 'inside.txt') + write_lib(cls.tmp, {'inside.txt': B_LINES * 2 + A_LINES}) + # ... and with many words the lexicon never saw + words = made_up(60) + cls.outside = os.path.join(cls.tmp, 'outside.txt') + write_lib(cls.tmp, {'outside.txt': A_LINES + [ + ' '.join(words[i:i + 6]) for i in range(0, len(words), 6)]}) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp, ignore_errors=True) + + def scan(self, path, log=None, **kw): + cfg = small_cfg(**{'book_domain_min_words': 10, + 'book_domain_min_types': 5, + 'book_domain_min_lexicon': 0, **kw}) + res = book_scan.scan_book(self.out, 'file', path, cfg=cfg, + progress=(log.append if log is not None + else None)) + res.pop('seconds') + return res + + @staticmethod + def types(res): + return Counter(r['errtype'] for r in res['findings']) + + def test_off_by_default_result_unchanged(self): + self.assertEqual(Config().book_domain_max, 0) + res = self.scan(self.outside) + self.assertNotIn('domain', res) + self.assertEqual(res, self.scan(self.outside, book_domain_max=0)) + + def test_inside_domain_nothing_changes(self): + off = self.scan(self.inside) + log = [] + on = self.scan(self.inside, log, book_domain_max=0.10) + dom = on.pop('domain') + self.assertFalse(dom['out']) + self.assertEqual(dom['left_out'], {}) + self.assertEqual(on, off) + self.assertTrue(any('בתוך תחום המילון' in s for s in log)) + + def test_outside_domain_rarity_findings_left_out(self): + off = self.scan(self.outside) + log = [] + on = self.scan(self.outside, log, book_domain_max=0.10) + dom = on['domain'] + self.assertTrue(dom['out']) + self.assertGreater(dom['share'], 0.10) + before, after = self.types(off), self.types(on) + # the fixture's book has rarity-based findings to lose ... + self.assertTrue(any(t in before for t in book_scan.DOMAIN_TYPES)) + for t in book_scan.DOMAIN_TYPES: + self.assertNotIn(t, after) + self.assertEqual(dom['left_out'].get(t, 0), before.get(t, 0)) + # ... and every other kind stays, row for row + kept = [r for r in off['findings'] + if r['errtype'] not in book_scan.DOMAIN_TYPES] + self.assertTrue(kept) + self.assertIn('final_midword', after) + self.assertEqual(on['findings'], kept) + self.assertEqual(on['space_errors'], off['space_errors']) + # the scan says so, by name + self.assertTrue(any('מחוץ לתחום המילון' in s + and 'book_domain_max' in s for s in log)) + + def test_minimum_words(self): + on = self.scan(self.outside, book_domain_max=0.10, + book_domain_min_words=10 ** 6) + self.assertFalse(on['domain']['out']) + on.pop('domain') + self.assertEqual(on, self.scan(self.outside)) + + def test_split_alternatives_follow_the_gate(self): + off = self.scan(self.outside) + on = self.scan(self.outside, book_domain_max=0.10) + self.assertTrue(on['domain']['out']) + self.assertTrue(on['domain']['left_out'].get('missing_space')) + self.assertTrue([a for a in off['split_alternatives'] + if a['is_primary']]) + # the missing_space finding is gone: no primary alternative, and no + # alternative of any word whose findings were all left out + self.assertFalse([a for a in on['split_alternatives'] + if a['is_primary']]) + shown = {r['word'] for r in on['findings']} + gone = {r['word'] for r in off['findings']} - shown + self.assertTrue(gone) + self.assertFalse({a['word'] for a in on['split_alternatives']} & gone) + # inside the domain nothing is filtered + self.assertEqual(self.scan(self.inside)['split_alternatives'], + self.scan(self.inside, + book_domain_max=0.10)['split_alternatives']) + + def test_small_lexicon_refuses_the_gate(self): + off = self.scan(self.outside) + log = [] + on = self.scan(self.outside, log, book_domain_max=0.10, + book_domain_min_lexicon=10 ** 9) + dom = on.pop('domain') + self.assertFalse(dom['out']) + self.assertEqual(dom['refused'], 'small') + self.assertEqual(dom['left_out'], {}) + self.assertGreater(dom['share'], 0.10) # it would have gated + self.assertEqual(on, off) + self.assertTrue(any('book_domain_min_lexicon' in s for s in log)) + # the default floor refuses this tiny test library too + floor = Config().book_domain_min_lexicon + self.assertEqual(self.scan( + self.outside, book_domain_max=0.10, + book_domain_min_lexicon=floor)['domain']['refused'], 'small') + + def test_partial_lexicon_refuses_the_gate(self): + off = self.scan(self.outside) + log = [] + with mock.patch.object(book_scan, 'domain_lexicon_problem', + return_value='partial'): + on = self.scan(self.outside, log, book_domain_max=0.10) + self.assertEqual(on.pop('domain')['refused'], 'partial') + self.assertEqual(on, off) + self.assertTrue(any('--allow-unread' in s for s in log)) + + def test_higher_threshold_keeps_the_book(self): + share = self.scan(self.outside, + book_domain_max=0.10)['domain']['share'] + on = self.scan(self.outside, book_domain_max=share + 0.001) + self.assertFalse(on['domain']['out']) + + +class Wiring(unittest.TestCase): + """The settings the web UI shows and passes on.""" + + def test_ui_schema_and_overrides(self): + tmp = tempfile.mkdtemp(prefix='magiah_domain_ui_') + self.addCleanup(shutil.rmtree, tmp, True) + fields = {f['key']: f for f in scanner.scan_config(tmp)['fields']} + for key, typ, default in ( + ('book_domain_max', 'float', 0.0), + ('book_domain_min_words', 'int', 200), + ('book_domain_min_types', 'int', 100), + ('book_domain_min_lexicon', 'int', 1000000)): + f = fields[key] + self.assertEqual((f['type'], f['default']), (typ, default), key) + self.assertNotEqual(f['hebrew'], key) + self.assertTrue(f['explanation']) + cfg = scanner._merge_config(tmp, {'book_domain_max': '0.05', + 'book_domain_min_types': '50'}) + self.assertEqual((cfg.book_domain_max, cfg.book_domain_min_types), + (0.05, 50)) + + +if __name__ == '__main__': + unittest.main()