diff --git a/CHANGELOG.md b/CHANGELOG.md index acf60cdc..4475384b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Every release has a **Change lists** heading. It names each change that makes a ### Fixed +- `gwt-read genae search --locality` now checks the value against the site's search form for the province: a listed name becomes its id, and an id is confirmed to belong to that province. Before, a name or an unknown id went out as `l=` and the site silently ignored it, so the search looked filtered but returned the whole province. An unknown, ambiguous or blank value now exits 64 (`--locality ''` used to search the whole province), and the run prints which locality it searched (#246). - `gwt-read pares search` no longer pauses every lane when one query gets an empty or unreadable page. It reads the source's canary once, live, through the gate; when the canary answers with at least its minimum, the search exits 7 (`unreadable`, new), saves and caches nothing for that query, and the gate only logs it. A canary that is empty, short, walled, unreachable or the search itself keeps the wall (exit 3). Record an exit 7 as `unproven`, never `none` (#237). ## [1.3.2] - 2026-11-07 diff --git a/docs/read.md b/docs/read.md index 11f69cd5..bc94261b 100644 --- a/docs/read.md +++ b/docs/read.md @@ -211,7 +211,7 @@ gwt read genae search --given Ana --surname Zorvalo --province mu --out runs/ | --- | --- | | `--given`, `--surname` (twice for both surnames) | the name as the site's `n`, `a1`, `a2` fields; at least one is needed | | `--province` | `mu` (default), `ab`, `al`, `am`, `gr` | -| `--locality` | the site's numeric locality id (its own `l=` field); with it the site leaves out the place column, so candidates carry no place | +| `--locality` | the site's numeric locality id (its own `l=` field), or its listed name. Both are checked against the site's search form for `--province` first (the form is cached and never refetched by `--refresh`), and the run prints `locality`. A blank value, an id the province does not list, or a name with no exact match (case, accents, hyphens and spacing aside) stops the search with exit 64; a name is never taken as a prefix, and the near names are offered instead. A form that lists no locality for the province exits 1: the site changed its form. With the flag left out the search is regional on purpose, because the site silently ignores an `l` it does not know. With a locality the site leaves out the place column, so candidates carry no place | | `--from`, `--to` | year range | Each row becomes a candidate. A baptism gives the child with the parents' first names. A marriage gives two candidates, one per spouse, with the full name only (the site does not split it) and the other spouse as a relative. A burial gives a death event. A marriage both spouses match appears in the groom and the bride blocks; it is written once. Rows the site marks as phonetic suggestions carry `phonetic: true`. gwt-rank reads neither the marriage event nor the phonetic mark, so a groom or bride ranks on names and spouse only, and the lane reads `phonetic` itself. Only the first page of each block is read (15 rows); a block that fills it prints as `15+ (cut: narrow the search)`. The gate keeps one request every 20 seconds at best (`PACE` in `read/gwt_genae.py`), so one search costs about a minute. Searches stay in the cache for 7 days (`GENAE_RULES`); when all four blocks come from the cache, the `results` line ends in ` (cache)` and the evidence has `from_cache: true`. diff --git a/read/gwt_genae.py b/read/gwt_genae.py index a518fbcf..9625f61b 100644 --- a/read/gwt_genae.py +++ b/read/gwt_genae.py @@ -14,7 +14,7 @@ it is flagged as cut, and the lane narrows the search instead of paging. A block the site answers with "no indexed records for these years in this place" is a real answer with no rows. Standard library only. """ -import html, re, urllib.parse +import html, re, unicodedata, urllib.parse HOST = 'https://genae.org' SOURCE = 'genae' @@ -26,6 +26,7 @@ LANGUAGE = 'es-ES,es;q=0.9' BLOCKS = ('bautismos', 'casados', 'casadas', 'defunciones') PROVINCES = ('mu', 'ab', 'al', 'am', 'gr') +LOCALITIES_URL = HOST + '/buscar' # its search form lists every locality as ', re.S) +_ATTR = re.compile(r'([\w-]+)="([^"]*)"') + + +class LocalityError(ValueError): + """A --locality the site does not list under the province: an unknown or ambiguous name, or an unknown id.""" + + +class LocalityFormError(RuntimeError): + """The search form lists no locality for the province: the site changed its form, not the user's typo.""" + + +def fold(name): + """Name folding for matching: no accents, lower, hyphens as spaces, single spaces.""" + return re.sub(r'[\s-]+', ' ', ''.join(c for c in unicodedata.normalize('NFKD', name or '') + if not unicodedata.combining(c)).casefold()).strip() + + +def _options(page): + """(id, province, label) of each locality ' + b'' + b'' + b'' + b'' + b'') + + +class Localities(unittest.TestCase): + MAP = gwt_genae.locality_map(LOCALITIES.decode()) + + def test_map_parses_listed_options_by_province(self): + self.assertEqual(self.MAP['mu'], [('701', 'Villazul'), ('702', 'Peñafria (Norte)'), ('703', 'Peñafria (Sur)')]) + self.assertEqual(self.MAP['ab'], [('801', 'Villarreal de Arriba')]) + self.assertTrue(gwt_genae.is_locality_list(LOCALITIES.decode())) + self.assertFalse(gwt_genae.is_locality_list(PAGE['bautismos'].decode())) + + def test_a_listed_numeric_id_resolves_to_itself_and_its_name(self): + self.assertEqual(gwt_genae.resolve_locality('701', 'mu', self.MAP), ('701', 'Villazul')) + self.assertEqual(gwt_genae.resolve_locality(' 701 ', 'mu', self.MAP), ('701', 'Villazul')) + + def test_an_unknown_id_or_an_id_of_another_province_stops(self): + # the site ignores an l it does not know: an unchecked id is the same silent regional search as a name + for lid in ('999', '801'): + with self.assertRaises(gwt_genae.LocalityError): + gwt_genae.resolve_locality(lid, 'mu', self.MAP) + + def test_a_name_resolves_ignoring_case_accents_and_spaces(self): + self.assertEqual(gwt_genae.resolve_locality('villazul', 'mu', self.MAP), ('701', 'Villazul')) + self.assertEqual(gwt_genae.resolve_locality(' Peñafría (norte) ', 'mu', self.MAP)[0], '702') + + def test_a_hyphen_folds_like_a_space(self): + self.assertEqual(gwt_genae.fold('Torre-Mayor'), gwt_genae.fold('torre mayor')) + + def test_a_unique_prefix_is_never_taken_only_suggested(self): + # a place the site does not list must not silently become a longer listed name + with self.assertRaises(gwt_genae.LocalityError) as ctx: + gwt_genae.resolve_locality('Villa', 'mu', self.MAP) + self.assertIn('did you mean: 701 Villazul', str(ctx.exception)) + + def test_two_options_with_one_name_stop_and_list_their_ids(self): + page = ('' + '') + with self.assertRaises(gwt_genae.LocalityError) as ctx: + gwt_genae.resolve_locality('Villazul', 'mu', gwt_genae.locality_map(page)) + self.assertIn('711 Villazul', str(ctx.exception)) + self.assertIn('712 Villazul', str(ctx.exception)) + + def test_options_parse_whatever_their_attribute_order(self): + page = '' + self.assertTrue(gwt_genae.is_locality_list(page)) + self.assertEqual(gwt_genae.locality_map(page), {'mu': [('701', 'Villazul')]}) + + def test_an_option_without_a_numeric_id_is_not_a_locality(self): + page = '' + self.assertFalse(gwt_genae.is_locality_list(page)) + self.assertEqual(gwt_genae.locality_map(page), {}) + + def test_a_province_the_form_does_not_list_is_a_changed_form_not_a_typo(self): + with self.assertRaises(gwt_genae.LocalityFormError): + gwt_genae.resolve_locality('Villazul', 'gr', self.MAP) + + def test_a_name_of_another_province_is_unknown(self): + with self.assertRaises(gwt_genae.LocalityError): + gwt_genae.resolve_locality('Villarreal de Arriba', 'mu', self.MAP) + + def test_an_ambiguous_name_stops_and_lists_the_options(self): + with self.assertRaises(gwt_genae.LocalityError) as ctx: + gwt_genae.resolve_locality('Peñafria', 'mu', self.MAP) + self.assertIn('Peñafria (Norte)', str(ctx.exception)) + self.assertIn('Peñafria (Sur)', str(ctx.exception)) + + def test_an_unknown_name_stops(self): + with self.assertRaises(gwt_genae.LocalityError): + gwt_genae.resolve_locality('Noloconozco', 'mu', self.MAP) + + class Parse(unittest.TestCase): def test_params_use_the_sites_own_names_and_skip_empty_ones(self): self.assertEqual(gwt_genae.params('Ana', 'Zorvalo', '', 'mu', '', 1850, 1900), @@ -94,6 +173,88 @@ def test_only_the_block_asked_for_is_an_answer(self): class Search(Genae): + def test_a_named_locality_is_resolved_before_the_search(self): + gate = FakeGate() + net = FakeNet(gate, (200, LOCALITIES), *ANSWERS) + code, lines, text = self.search(net, gate, ('--locality', 'Villazul')) + self.assertEqual(code, 0, text) + self.assertEqual(len(net.requests), 5) # the form, then the four blocks + for req in net.requests[1:]: + self.assertIn('l=701', req.full_url) + self.assertEqual(lines['locality'], '701\tVillazul') # the evidence shows which town was searched + self.assertIsNone(net.requests[0].get_header('Hx-request')) # the full form, never an htmx fragment + + def test_a_numeric_locality_is_checked_against_the_form(self): + gate = FakeGate() + net = FakeNet(gate, (200, LOCALITIES), *ANSWERS) + code, lines, text = self.search(net, gate, ('--locality', '701')) + self.assertEqual(code, 0, text) + self.assertEqual(len(net.requests), 5) + self.assertEqual(lines['locality'], '701\tVillazul') + + def test_an_unknown_numeric_locality_stops_before_any_search_request(self): + gate = FakeGate() + net = FakeNet(gate, (200, LOCALITIES)) + code, _, text = self.search(net, gate, ('--locality', '801')) # listed, but under ab + self.assertEqual(code, 64, text) + self.assertEqual(len(net.requests), 1) + + def test_a_form_without_the_province_is_a_source_error_not_a_usage_error(self): + gate = FakeGate() + net = FakeNet(gate, (200, LOCALITIES)) + code, _, text = self.search(net, gate, ('--province', 'gr', '--locality', 'Villazul')) + self.assertEqual(code, 1, text) + self.assertIn('changed its form', text) + self.assertEqual(len(net.requests), 1) + + def test_refresh_reads_the_locality_form_from_the_cache(self): + gate = FakeGate() + net = FakeNet(gate, (200, LOCALITIES), *ANSWERS) + self.assertEqual(self.search(net, gate, ('--locality', 'Villazul'))[0], 0) + gate = FakeGate() + net = FakeNet(gate, *ANSWERS) + o, e = io.StringIO(), io.StringIO() + argv = ['genae', '--cache-dir', self.cache, '--lane', 't', '--refresh', 'search', '--given', 'Ana', + '--surname', 'Zorvalo', '--locality', 'Villazul', '--out', self.out] + with contextlib.redirect_stdout(o), contextlib.redirect_stderr(e): + code = gwt_read.main(argv, urlopen=net, gate=gate) + text = o.getvalue() + e.getvalue() + self.assertEqual(code, 0, text) + self.assertEqual(len(net.requests), 4) # the four blocks again; the form came from the cache + + def test_a_blank_locality_stops_before_any_request(self): + gate = FakeGate() + net = FakeNet(gate) + code, _, text = self.search(net, gate, ('--locality', ' ')) + self.assertEqual(code, 64) + self.assertIn('blank', text) + self.assertEqual(net.requests, []) # not even the form was asked for + + def test_an_empty_locality_stops_before_any_request(self): + gate = FakeGate() + net = FakeNet(gate) + code, _, text = self.search(net, gate, ('--locality', '')) + self.assertEqual(code, 64) + self.assertIn('blank', text) + self.assertEqual(net.requests, []) + + def test_an_omitted_locality_searches_regionally_on_purpose(self): + gate = FakeGate() + net = FakeNet(gate, *ANSWERS) + code, lines, text = self.search(net, gate) + self.assertEqual(code, 0, text) + self.assertEqual(len(net.requests), 4) + for req in net.requests: + self.assertNotIn('l=', req.full_url) + + def test_an_unknown_locality_stops_before_any_search_request(self): + gate = FakeGate() + net = FakeNet(gate, (200, LOCALITIES)) + code, _, text = self.search(net, gate, ('--locality', 'Noloconozco')) + self.assertEqual(code, 64) + self.assertIn('not a listed locality', text) + self.assertEqual(len(net.requests), 1) # only the form; no block was asked for + def test_search_reads_the_four_blocks_and_writes_candidates_gwt_rank_ranks(self): gate = FakeGate() net = FakeNet(gate, *ANSWERS)