From e2ef7c6e2884940c47467015fb071883f2e80571 Mon Sep 17 00:00:00 2001 From: Instinct Agent Date: Fri, 6 Nov 2026 22:19:43 +0000 Subject: [PATCH 1/3] fix(read): improve genae query handling --- docs/read.md | 2 +- read/gwt_genae.py | 59 ++++++++++++++++++++- read/gwt_read.py | 16 +++++- read/mutants.d/genae-locality.txt | 2 + read/test_gwt_genae.py | 88 +++++++++++++++++++++++++++++++ 5 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 read/mutants.d/genae-locality.txt diff --git a/docs/read.md b/docs/read.md index ad1046bb..29c11eb1 100644 --- a/docs/read.md +++ b/docs/read.md @@ -208,7 +208,7 @@ gwt read genae search --given Ana --surname Zorvalo --province mu --out runs/ | --- | --- | | `--given`, `--surname` (twice for both surnames) | the name as the site's `n`, `a1`, `a2` fields; at least one is needed | | `--province` | `mu` (default), `ab`, `al`, `am`, `gr` | -| `--locality` | the site's numeric locality id (its own `l=` field); with it the site leaves out the place column, so candidates carry no place | +| `--locality` | the site's numeric locality id (its own `l=` field), or its listed name: the name is resolved against the site's search form first, and an unknown, blank (including an empty value) or ambiguous name stops the search with exit 64; with the flag left out the search is regional on purpose, because the site silently ignores an `l` it does not know. With a locality the site leaves out the place column, so candidates carry no place | | `--from`, `--to` | year range | Each row becomes a candidate. A baptism gives the child with the parents' first names. A marriage gives two candidates, one per spouse, with the full name only (the site does not split it) and the other spouse as a relative. A burial gives a death event. A marriage both spouses match appears in the groom and the bride blocks; it is written once. Rows the site marks as phonetic suggestions carry `phonetic: true`. gwt-rank reads neither the marriage event nor the phonetic mark, so a groom or bride ranks on names and spouse only, and the lane reads `phonetic` itself. Only the first page of each block is read (15 rows); a block that fills it prints as `15+ (cut: narrow the search)`. The gate keeps one request every 20 seconds at best (`PACE` in `read/gwt_genae.py`), so one search costs about a minute. Searches stay in the cache for 7 days (`GENAE_RULES`). diff --git a/read/gwt_genae.py b/read/gwt_genae.py index a518fbcf..a686315d 100644 --- a/read/gwt_genae.py +++ b/read/gwt_genae.py @@ -14,7 +14,7 @@ it is flagged as cut, and the lane narrows the search instead of paging. A block the site answers with "no indexed records for these years in this place" is a real answer with no rows. Standard library only. """ -import html, re, urllib.parse +import html, re, unicodedata, urllib.parse HOST = 'https://genae.org' SOURCE = 'genae' @@ -26,6 +26,7 @@ LANGUAGE = 'es-ES,es;q=0.9' BLOCKS = ('bautismos', 'casados', 'casadas', 'defunciones') PROVINCES = ('mu', 'ab', 'al', 'am', 'gr') +LOCALITIES_URL = HOST + '/buscar' # its search form lists every locality as ', re.S) + + +class LocalityError(ValueError): + """A --locality that is not the site's numeric id and no listed name resolves to.""" + + +def fold(name): + """Name folding for matching: no accents, lower, single spaces.""" + return re.sub(r'\s+', ' ', ''.join(c for c in unicodedata.normalize('NFKD', name or '') + if not unicodedata.combining(c)).casefold()).strip() + + +def is_locality_list(page): + """True for the site's search form, which lists every locality as ' + b'' + b'' + b'' + b'' + b'') + + +class Localities(unittest.TestCase): + MAP = gwt_genae.locality_map(LOCALITIES.decode()) + + def test_map_parses_listed_options_by_province(self): + self.assertEqual(self.MAP['mu'], [('701', 'Villazul'), ('702', 'Peñafria (Norte)'), ('703', 'Peñafria (Sur)')]) + self.assertEqual(self.MAP['ab'], [('801', 'Villarreal de Arriba')]) + self.assertTrue(gwt_genae.is_locality_list(LOCALITIES.decode())) + self.assertFalse(gwt_genae.is_locality_list(PAGE['bautismos'].decode())) + + def test_a_numeric_id_passes_through(self): + self.assertEqual(gwt_genae.resolve_locality('701', 'mu', self.MAP), '701') + self.assertEqual(gwt_genae.resolve_locality('', 'mu', self.MAP), '') + + def test_a_name_resolves_ignoring_case_accents_and_spaces(self): + self.assertEqual(gwt_genae.resolve_locality('villazul', 'mu', self.MAP), '701') + self.assertEqual(gwt_genae.resolve_locality(' Peñafría (norte) ', 'mu', self.MAP), '702') + + def test_a_name_of_another_province_is_unknown(self): + with self.assertRaises(gwt_genae.LocalityError): + gwt_genae.resolve_locality('Villarreal de Arriba', 'mu', self.MAP) + + def test_an_ambiguous_name_stops_and_lists_the_options(self): + with self.assertRaises(gwt_genae.LocalityError) as ctx: + gwt_genae.resolve_locality('Peñafria', 'mu', self.MAP) + self.assertIn('Peñafria (Norte)', str(ctx.exception)) + self.assertIn('Peñafria (Sur)', str(ctx.exception)) + + def test_an_unknown_name_stops(self): + with self.assertRaises(gwt_genae.LocalityError): + gwt_genae.resolve_locality('Noloconozco', 'mu', self.MAP) + + def test_a_blank_value_stops(self): + with self.assertRaises(gwt_genae.LocalityError): + gwt_genae.resolve_locality(' ', 'mu', self.MAP) + + class Parse(unittest.TestCase): def test_params_use_the_sites_own_names_and_skip_empty_ones(self): self.assertEqual(gwt_genae.params('Ana', 'Zorvalo', '', 'mu', '', 1850, 1900), @@ -94,6 +140,48 @@ def test_only_the_block_asked_for_is_an_answer(self): class Search(Genae): + def test_a_named_locality_is_resolved_before_the_search(self): + gate = FakeGate() + net = FakeNet(gate, (200, LOCALITIES), *ANSWERS) + code, lines, text = self.search(net, gate, ('--locality', 'Villazul')) + self.assertEqual(code, 0, text) + self.assertEqual(len(net.requests), 5) # the form, then the four blocks + for req in net.requests[1:]: + self.assertIn('l=701', req.full_url) + + def test_a_blank_locality_stops_before_any_request(self): + gate = FakeGate() + net = FakeNet(gate) + code, _, text = self.search(net, gate, ('--locality', ' ')) + self.assertEqual(code, 64) + self.assertIn('blank', text) + self.assertEqual(net.requests, []) # not even the form was asked for + + def test_an_empty_locality_stops_before_any_request(self): + gate = FakeGate() + net = FakeNet(gate) + code, _, text = self.search(net, gate, ('--locality', '')) + self.assertEqual(code, 64) + self.assertIn('blank', text) + self.assertEqual(net.requests, []) + + def test_an_omitted_locality_searches_regionally_on_purpose(self): + gate = FakeGate() + net = FakeNet(gate, *ANSWERS) + code, lines, text = self.search(net, gate) + self.assertEqual(code, 0, text) + self.assertEqual(len(net.requests), 4) + for req in net.requests: + self.assertNotIn('l=', req.full_url) + + def test_an_unknown_locality_stops_before_any_search_request(self): + gate = FakeGate() + net = FakeNet(gate, (200, LOCALITIES)) + code, _, text = self.search(net, gate, ('--locality', 'Noloconozco')) + self.assertEqual(code, 64) + self.assertIn('not a listed locality', text) + self.assertEqual(len(net.requests), 1) # only the form; no block was asked for + def test_search_reads_the_four_blocks_and_writes_candidates_gwt_rank_ranks(self): gate = FakeGate() net = FakeNet(gate, *ANSWERS) From eb39e55cfb939429a8bb4ba35d9558d5c5b01d75 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:42:15 +0100 Subject: [PATCH 2/3] fix(read): never let a genae locality silently become another town Take only an exact folded name; offer prefix matches as suggestions instead of searching them. Check numeric ids against the province's listed options too, since the site ignores an l it does not know. A form with no options for the province exits 1 (the site changed), not 64. Print the resolved locality. Read the form from the cache even under --refresh, without the hx-request header, and parse option attributes in any order. Fold hyphens like spaces, list ids in an ambiguity message, drop the unreachable blank check, and add the CHANGELOG entry. --- CHANGELOG.md | 4 ++ docs/read.md | 2 +- read/gwt_genae.py | 68 +++++++++++++++---------- read/gwt_read.py | 17 ++++--- read/mutants.d/genae-locality.txt | 3 +- read/test_gwt_genae.py | 85 +++++++++++++++++++++++++++---- 6 files changed, 135 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 707c40cd..3b68956b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ Every release has a **Change lists** heading. It names each change that makes a ## [Unreleased] +### Fixed + +- `gwt-read genae search --locality` now checks the value against the site's search form for the province: a listed name becomes its id, and an id is confirmed to belong to that province. Before, a name or an unknown id went out as `l=` and the site silently ignored it, so the search looked filtered but returned the whole province. An unknown, ambiguous or blank value now exits 64 (`--locality ''` used to search the whole province), and the run prints which locality it searched (#246). + ## [1.3.1] - 2026-11-06 ### Change lists diff --git a/docs/read.md b/docs/read.md index 29c11eb1..d7801beb 100644 --- a/docs/read.md +++ b/docs/read.md @@ -208,7 +208,7 @@ gwt read genae search --given Ana --surname Zorvalo --province mu --out runs/ | --- | --- | | `--given`, `--surname` (twice for both surnames) | the name as the site's `n`, `a1`, `a2` fields; at least one is needed | | `--province` | `mu` (default), `ab`, `al`, `am`, `gr` | -| `--locality` | the site's numeric locality id (its own `l=` field), or its listed name: the name is resolved against the site's search form first, and an unknown, blank (including an empty value) or ambiguous name stops the search with exit 64; with the flag left out the search is regional on purpose, because the site silently ignores an `l` it does not know. With a locality the site leaves out the place column, so candidates carry no place | +| `--locality` | the site's numeric locality id (its own `l=` field), or its listed name. Both are checked against the site's search form for `--province` first (the form is cached and never refetched by `--refresh`), and the run prints `locality`. A blank value, an id the province does not list, or a name with no exact match (case, accents, hyphens and spacing aside) stops the search with exit 64; a name is never taken as a prefix, and the near names are offered instead. A form that lists no locality for the province exits 1: the site changed its form. With the flag left out the search is regional on purpose, because the site silently ignores an `l` it does not know. With a locality the site leaves out the place column, so candidates carry no place | | `--from`, `--to` | year range | Each row becomes a candidate. A baptism gives the child with the parents' first names. A marriage gives two candidates, one per spouse, with the full name only (the site does not split it) and the other spouse as a relative. A burial gives a death event. A marriage both spouses match appears in the groom and the bride blocks; it is written once. Rows the site marks as phonetic suggestions carry `phonetic: true`. gwt-rank reads neither the marriage event nor the phonetic mark, so a groom or bride ranks on names and spouse only, and the lane reads `phonetic` itself. Only the first page of each block is read (15 rows); a block that fills it prints as `15+ (cut: narrow the search)`. The gate keeps one request every 20 seconds at best (`PACE` in `read/gwt_genae.py`), so one search costs about a minute. Searches stay in the cache for 7 days (`GENAE_RULES`). diff --git a/read/gwt_genae.py b/read/gwt_genae.py index a686315d..9625f61b 100644 --- a/read/gwt_genae.py +++ b/read/gwt_genae.py @@ -39,56 +39,70 @@ def text(fragment): return re.sub(r'\s+', ' ', t).strip() -_OPTION = re.compile(r'', re.S) +_OPTION = re.compile(r']*)>(.*?)', re.S) +_ATTR = re.compile(r'([\w-]+)="([^"]*)"') class LocalityError(ValueError): - """A --locality that is not the site's numeric id and no listed name resolves to.""" + """A --locality the site does not list under the province: an unknown or ambiguous name, or an unknown id.""" + + +class LocalityFormError(RuntimeError): + """The search form lists no locality for the province: the site changed its form, not the user's typo.""" def fold(name): - """Name folding for matching: no accents, lower, single spaces.""" - return re.sub(r'\s+', ' ', ''.join(c for c in unicodedata.normalize('NFKD', name or '') - if not unicodedata.combining(c)).casefold()).strip() + """Name folding for matching: no accents, lower, hyphens as spaces, single spaces.""" + return re.sub(r'[\s-]+', ' ', ''.join(c for c in unicodedata.normalize('NFKD', name or '') + if not unicodedata.combining(c)).casefold()).strip() + + +def _options(page): + """(id, province, label) of each locality