fix(read): resolve Genae locality names before searching

CHANGELOG.md

@@ -8,6 +8,7 @@ Every release has a **Change lists** heading. It names each change that makes a
 
 ### Fixed
 
+- `gwt-read genae search --locality` now checks the value against the site's search form for the province: a listed name becomes its id, and an id is confirmed to belong to that province. Before, a name or an unknown id went out as `l=` and the site silently ignored it, so the search looked filtered but returned the whole province. An unknown, ambiguous or blank value now exits 64 (`--locality ''` used to search the whole province), and the run prints which locality it searched (#246).
 - `gwt-read pares search` no longer pauses every lane when one query gets an empty or unreadable page. It reads the source's canary once, live, through the gate; when the canary answers with at least its minimum, the search exits 7 (`unreadable`, new), saves and caches nothing for that query, and the gate only logs it. A canary that is empty, short, walled, unreachable or the search itself keeps the wall (exit 3). Record an exit 7 as `unproven`, never `none` (#237).
 
 ## [1.3.2] - 2026-11-07

docs/read.md

@@ -211,7 +211,7 @@ gwt read genae search --given Ana --surname Zorvalo --province mu --out runs/
 | --- | --- |
 | `--given`, `--surname` (twice for both surnames) | the name as the site's `n`, `a1`, `a2` fields; at least one is needed |
 | `--province` | `mu` (default), `ab`, `al`, `am`, `gr` |
-| `--locality` | the site's numeric locality id (its own `l=` field); with it the site leaves out the place column, so candidates carry no place |
+| `--locality` | the site's numeric locality id (its own `l=` field), or its listed name. Both are checked against the site's search form for `--province` first (the form is cached and never refetched by `--refresh`), and the run prints `locality<TAB><id><TAB><name>`. A blank value, an id the province does not list, or a name with no exact match (case, accents, hyphens and spacing aside) stops the search with exit 64; a name is never taken as a prefix, and the near names are offered instead. A form that lists no locality for the province exits 1: the site changed its form. With the flag left out the search is regional on purpose, because the site silently ignores an `l` it does not know. With a locality the site leaves out the place column, so candidates carry no place |
 | `--from`, `--to` | year range |
 
 Each row becomes a candidate. A baptism gives the child with the parents' first names. A marriage gives two candidates, one per spouse, with the full name only (the site does not split it) and the other spouse as a relative. A burial gives a death event. A marriage both spouses match appears in the groom and the bride blocks; it is written once. Rows the site marks as phonetic suggestions carry `phonetic: true`. gwt-rank reads neither the marriage event nor the phonetic mark, so a groom or bride ranks on names and spouse only, and the lane reads `phonetic` itself. Only the first page of each block is read (15 rows); a block that fills it prints as `15+ (cut: narrow the search)`. The gate keeps one request every 20 seconds at best (`PACE` in `read/gwt_genae.py`), so one search costs about a minute. Searches stay in the cache for 7 days (`GENAE_RULES`); when all four blocks come from the cache, the `results` line ends in ` (cache)` and the evidence has `from_cache: true`.

read/gwt_genae.py

@@ -14,7 +14,7 @@
 it is flagged as cut, and the lane narrows the search instead of paging. A block the site answers with "no indexed
 records for these years in this place" is a real answer with no rows. Standard library only.
 """
-import html, re, urllib.parse
+import html, re, unicodedata, urllib.parse
 
 HOST = 'https://genae.org'
 SOURCE = 'genae'
@@ -26,6 +26,7 @@
 LANGUAGE = 'es-ES,es;q=0.9'
 BLOCKS = ('bautismos', 'casados', 'casadas', 'defunciones')
 PROVINCES = ('mu', 'ab', 'al', 'am', 'gr')
+LOCALITIES_URL = HOST + '/buscar'   # its search form lists every locality as <option value=id>
 
 _ROW = re.compile(r'<tr class="([^"]*)" data-registro-tipo="([^"]*)" data-registro-id="(\d+)" data-registro-ano="(\d*)"[^>]*>(.*?)</tr>', re.S)
 _CELL = re.compile(r'<td data-col="([^"]+)"[^>]*>(.*?)</td>', re.S)
@@ -38,8 +39,76 @@ def text(fragment):
     return re.sub(r'\s+', ' ', t).strip()
 
 
+_OPTION = re.compile(r'<option\b([^>]*)>(.*?)</option>', re.S)
+_ATTR = re.compile(r'([\w-]+)="([^"]*)"')
+
+
+class LocalityError(ValueError):
+    """A --locality the site does not list under the province: an unknown or ambiguous name, or an unknown id."""
+
+
+class LocalityFormError(RuntimeError):
+    """The search form lists no locality for the province: the site changed its form, not the user's typo."""
+
+
+def fold(name):
+    """Name folding for matching: no accents, lower, hyphens as spaces, single spaces."""
+    return re.sub(r'[\s-]+', ' ', ''.join(c for c in unicodedata.normalize('NFKD', name or '')
+                                          if not unicodedata.combining(c)).casefold()).strip()
+
+
+def _options(page):
+    """(id, province, label) of each locality <option>, its attributes in any order."""
+    for attrs, label in _OPTION.findall(page or ''):
+        a = dict(_ATTR.findall(attrs))
+        if re.fullmatch(r'[0-9]+', a.get('value', '')) and re.fullmatch(r'\w+', a.get('data-provincia', '')):
+            yield a['value'], a['data-provincia'], label
+
+
+def is_locality_list(page):
+    """True for the site's search form, which lists every locality as <option value=id data-provincia=..>."""
+    return any(True for _ in _options(page))
+
+
+def locality_map(page):
+    """{province: [(id, label)]} from the search form's locality <select>."""
+    out = {}
+    for lid, prov, label in _options(page):
+        out.setdefault(prov, []).append((lid, text(label)))
+    return out
+
+
+def resolve_locality(value, province, localities):
+    """(id, label) of the one locality of the province that --locality names: its numeric id, or its listed name
+    (folded, exact match only). Anything else raises LocalityError, naming the near names; a province the form
+    does not list raises LocalityFormError. The site never errors on an l it does not know, it silently returns the
+    whole region, so an unknown name or id must stop the search here, not there. A prefix is never taken: a place
+    the site does not list would silently become a longer listed name."""
+    value = value.strip()
+    listed = localities.get(province)
+    if not listed:
+        raise LocalityFormError('the search form lists no locality for %s: the site changed its form' % province)
+    if re.fullmatch(r'[0-9]+', value):
+        found = [(lid, label) for lid, label in listed if lid == value]
+        if found:
+            return found[0]
+        raise LocalityError('%s is not a listed locality id of %s' % (value, province))
+    folded = fold(value)
+    exact = [(lid, label) for lid, label in listed if fold(label) == folded]
+    if len(exact) == 1:
+        return exact[0]
+    if exact:
+        raise LocalityError('%r is ambiguous in %s: %s; give the id'
+                            % (value, province, ', '.join('%s %s' % o for o in exact)))
+    near = [(lid, label) for lid, label in listed if folded and fold(label).startswith(folded)]
+    raise LocalityError('%r is not a listed locality of %s%s' % (
+        value, province, '; did you mean: ' + ', '.join('%s %s' % o for o in near) if near else ''))
+
+
 def params(given='', surname1='', surname2='', province='mu', locality='', year_from=1503, year_to=1986):
-    """The site's own query: n, a1, a2 (each 'starts with'), r province, l locality, d and h year range."""
+    """The site's own query: n, a1, a2 (each 'starts with'), r province, l locality, d and h year range.
+    `locality` is the site's numeric id; the site silently ignores an l it does not know, so callers resolve
+    names with resolve_locality first (cmd_genae_search does)."""
     q = [('n', given), ('a1', surname1), ('a2', surname2), ('r', province)]
     if locality:
         q.append(('l', locality))

read/gwt_read.py

@@ -390,6 +390,23 @@ def cmd_genae_search(a, R):
     if not (a.given or a.surname):
         err('give --given, --surname or both')
         return EXIT_USAGE
+    if a.locality is not None and not a.locality.strip():
+        err("--locality is blank: give the site's numeric locality id or a listed name, or leave the flag out for a regional search")
+        return EXIT_USAGE
+    if a.locality:   # an id too: the site ignores an l it does not know, and an id belongs to one province
+        # the form lists the localities and rarely changes: the cache's own age, never --refresh
+        _st, _rh, mbody, _cached = R.get(gwt_genae.LOCALITIES_URL, {'accept': gwt_genae.ACCEPT,
+                                         'accept-language': gwt_genae.LANGUAGE}, page=gwt_genae.is_locality_list)
+        try:
+            a.locality, label = gwt_genae.resolve_locality(a.locality, a.province,
+                                                           gwt_genae.locality_map(mbody.decode('utf-8', 'replace')))
+        except gwt_genae.LocalityFormError as exc:
+            err(str(exc))
+            return EXIT_ERROR
+        except gwt_genae.LocalityError as exc:
+            err(str(exc))
+            return EXIT_USAGE
+        out('locality\t%s\t%s' % (a.locality, label))
     sur = (list(a.surname) + ['', ''])[:2]
     query = gwt_genae.params(a.given, sur[0], sur[1], a.province, a.locality, a.year_from, a.year_to)
     h = {'accept': gwt_genae.ACCEPT, 'accept-language': gwt_genae.LANGUAGE, 'hx-request': 'true'}
@@ -599,7 +616,7 @@ def _genae_search_args(s):
     s.add_argument('--given', default='')
     s.add_argument('--surname', action='append', default=[], help='first surname, then again for the second')
     s.add_argument('--province', default='mu', choices=gwt_genae.PROVINCES)
-    s.add_argument('--locality', default='', help="the site's numeric locality id (its own l= field)")
+    s.add_argument('--locality', default=None, help="the site's numeric locality id, or its listed name: resolved against the search form first; an unknown, blank or ambiguous value stops the search")
     s.add_argument('--from', dest='year_from', type=int, default=1503)
     s.add_argument('--to', dest='year_to', type=int, default=1986)
 

read/mutants.d/genae-locality.txt

@@ -0,0 +1,3 @@
+gwt_read.py |     if a.locality:   # an id too: the site ignores an l it does not know, and an id belongs to one province |     if False: | a genae locality, name or id, is checked against the search form before searching: an unknown one stops the search, never a silent whole-region search
+gwt_read.py |     if a.locality is not None and not a.locality.strip(): |     if False: | an explicitly blank --locality stops with exit 64 before any request; only an omitted flag searches regionally
+gwt_genae.py |     if not listed: |     if False: | a form that lists no locality for the province is a changed form (exit 1), never the user's typo (exit 64)

read/test_gwt_genae.py

@@ -30,6 +30,85 @@ def search(self, net, gate, extra=()):
         return self.run_genae(['--given', 'Ana', '--surname', 'Zorvalo'] + list(extra), net, gate)
 
 
+LOCALITIES = (b'<html><body><form id="form-busqueda" action="buscar">'
+              b'<select name="l">'
+              b'<option value="" selected>Todas</option>'
+              b'<option value="701" data-provincia="mu" data-desde="1700" data-hasta="1900">Villazul</option>'
+              b'<option value="702" data-provincia="mu">Pe\xc3\xb1afria (Norte)</option>'
+              b'<option value="703" data-provincia="mu">Pe\xc3\xb1afria (Sur)</option>'
+              b'<option value="801" data-provincia="ab">Villarreal de Arriba</option>'
+              b'</select></form></body></html>')
+
+
+class Localities(unittest.TestCase):
+    MAP = gwt_genae.locality_map(LOCALITIES.decode())
+
+    def test_map_parses_listed_options_by_province(self):
+        self.assertEqual(self.MAP['mu'], [('701', 'Villazul'), ('702', 'Peñafria (Norte)'), ('703', 'Peñafria (Sur)')])
+        self.assertEqual(self.MAP['ab'], [('801', 'Villarreal de Arriba')])
+        self.assertTrue(gwt_genae.is_locality_list(LOCALITIES.decode()))
+        self.assertFalse(gwt_genae.is_locality_list(PAGE['bautismos'].decode()))
+
+    def test_a_listed_numeric_id_resolves_to_itself_and_its_name(self):
+        self.assertEqual(gwt_genae.resolve_locality('701', 'mu', self.MAP), ('701', 'Villazul'))
+        self.assertEqual(gwt_genae.resolve_locality(' 701 ', 'mu', self.MAP), ('701', 'Villazul'))
+
+    def test_an_unknown_id_or_an_id_of_another_province_stops(self):
+        # the site ignores an l it does not know: an unchecked id is the same silent regional search as a name
+        for lid in ('999', '801'):
+            with self.assertRaises(gwt_genae.LocalityError):
+                gwt_genae.resolve_locality(lid, 'mu', self.MAP)
+
+    def test_a_name_resolves_ignoring_case_accents_and_spaces(self):
+        self.assertEqual(gwt_genae.resolve_locality('villazul', 'mu', self.MAP), ('701', 'Villazul'))
+        self.assertEqual(gwt_genae.resolve_locality('  Peñafría  (norte) ', 'mu', self.MAP)[0], '702')
+
+    def test_a_hyphen_folds_like_a_space(self):
+        self.assertEqual(gwt_genae.fold('Torre-Mayor'), gwt_genae.fold('torre mayor'))
+
+    def test_a_unique_prefix_is_never_taken_only_suggested(self):
+        # a place the site does not list must not silently become a longer listed name
+        with self.assertRaises(gwt_genae.LocalityError) as ctx:
+            gwt_genae.resolve_locality('Villa', 'mu', self.MAP)
+        self.assertIn('did you mean: 701 Villazul', str(ctx.exception))
+
+    def test_two_options_with_one_name_stop_and_list_their_ids(self):
+        page = ('<option value="711" data-provincia="mu">Villazul</option>'
+                '<option value="712" data-provincia="mu">Villazul</option>')
+        with self.assertRaises(gwt_genae.LocalityError) as ctx:
+            gwt_genae.resolve_locality('Villazul', 'mu', gwt_genae.locality_map(page))
+        self.assertIn('711 Villazul', str(ctx.exception))
+        self.assertIn('712 Villazul', str(ctx.exception))
+
+    def test_options_parse_whatever_their_attribute_order(self):
+        page = '<option data-provincia="mu" class="x" value="701">Villazul</option>'
+        self.assertTrue(gwt_genae.is_locality_list(page))
+        self.assertEqual(gwt_genae.locality_map(page), {'mu': [('701', 'Villazul')]})
+
+    def test_an_option_without_a_numeric_id_is_not_a_locality(self):
+        page = '<option value="todas" data-provincia="mu">Todas</option>'
+        self.assertFalse(gwt_genae.is_locality_list(page))
+        self.assertEqual(gwt_genae.locality_map(page), {})
+
+    def test_a_province_the_form_does_not_list_is_a_changed_form_not_a_typo(self):
+        with self.assertRaises(gwt_genae.LocalityFormError):
+            gwt_genae.resolve_locality('Villazul', 'gr', self.MAP)
+
+    def test_a_name_of_another_province_is_unknown(self):
+        with self.assertRaises(gwt_genae.LocalityError):
+            gwt_genae.resolve_locality('Villarreal de Arriba', 'mu', self.MAP)
+
+    def test_an_ambiguous_name_stops_and_lists_the_options(self):
+        with self.assertRaises(gwt_genae.LocalityError) as ctx:
+            gwt_genae.resolve_locality('Peñafria', 'mu', self.MAP)
+        self.assertIn('Peñafria (Norte)', str(ctx.exception))
+        self.assertIn('Peñafria (Sur)', str(ctx.exception))
+
+    def test_an_unknown_name_stops(self):
+        with self.assertRaises(gwt_genae.LocalityError):
+            gwt_genae.resolve_locality('Noloconozco', 'mu', self.MAP)
+
+
 class Parse(unittest.TestCase):
     def test_params_use_the_sites_own_names_and_skip_empty_ones(self):
         self.assertEqual(gwt_genae.params('Ana', 'Zorvalo', '', 'mu', '', 1850, 1900),
@@ -94,6 +173,88 @@ def test_only_the_block_asked_for_is_an_answer(self):
 
 
 class Search(Genae):
+    def test_a_named_locality_is_resolved_before_the_search(self):
+        gate = FakeGate()
+        net = FakeNet(gate, (200, LOCALITIES), *ANSWERS)
+        code, lines, text = self.search(net, gate, ('--locality', 'Villazul'))
+        self.assertEqual(code, 0, text)
+        self.assertEqual(len(net.requests), 5)   # the form, then the four blocks
+        for req in net.requests[1:]:
+            self.assertIn('l=701', req.full_url)
+        self.assertEqual(lines['locality'], '701\tVillazul')   # the evidence shows which town was searched
+        self.assertIsNone(net.requests[0].get_header('Hx-request'))   # the full form, never an htmx fragment
+
+    def test_a_numeric_locality_is_checked_against_the_form(self):
+        gate = FakeGate()
+        net = FakeNet(gate, (200, LOCALITIES), *ANSWERS)
+        code, lines, text = self.search(net, gate, ('--locality', '701'))
+        self.assertEqual(code, 0, text)
+        self.assertEqual(len(net.requests), 5)
+        self.assertEqual(lines['locality'], '701\tVillazul')
+
+    def test_an_unknown_numeric_locality_stops_before_any_search_request(self):
+        gate = FakeGate()
+        net = FakeNet(gate, (200, LOCALITIES))
+        code, _, text = self.search(net, gate, ('--locality', '801'))   # listed, but under ab
+        self.assertEqual(code, 64, text)
+        self.assertEqual(len(net.requests), 1)
+
+    def test_a_form_without_the_province_is_a_source_error_not_a_usage_error(self):
+        gate = FakeGate()
+        net = FakeNet(gate, (200, LOCALITIES))
+        code, _, text = self.search(net, gate, ('--province', 'gr', '--locality', 'Villazul'))
+        self.assertEqual(code, 1, text)
+        self.assertIn('changed its form', text)
+        self.assertEqual(len(net.requests), 1)
+
+    def test_refresh_reads_the_locality_form_from_the_cache(self):
+        gate = FakeGate()
+        net = FakeNet(gate, (200, LOCALITIES), *ANSWERS)
+        self.assertEqual(self.search(net, gate, ('--locality', 'Villazul'))[0], 0)
+        gate = FakeGate()
+        net = FakeNet(gate, *ANSWERS)
+        o, e = io.StringIO(), io.StringIO()
+        argv = ['genae', '--cache-dir', self.cache, '--lane', 't', '--refresh', 'search', '--given', 'Ana',
+                '--surname', 'Zorvalo', '--locality', 'Villazul', '--out', self.out]
+        with contextlib.redirect_stdout(o), contextlib.redirect_stderr(e):
+            code = gwt_read.main(argv, urlopen=net, gate=gate)
+        text = o.getvalue() + e.getvalue()
+        self.assertEqual(code, 0, text)
+        self.assertEqual(len(net.requests), 4)   # the four blocks again; the form came from the cache
+
+    def test_a_blank_locality_stops_before_any_request(self):
+        gate = FakeGate()
+        net = FakeNet(gate)
+        code, _, text = self.search(net, gate, ('--locality', '   '))
+        self.assertEqual(code, 64)
+        self.assertIn('blank', text)
+        self.assertEqual(net.requests, [])   # not even the form was asked for
+
+    def test_an_empty_locality_stops_before_any_request(self):
+        gate = FakeGate()
+        net = FakeNet(gate)
+        code, _, text = self.search(net, gate, ('--locality', ''))
+        self.assertEqual(code, 64)
+        self.assertIn('blank', text)
+        self.assertEqual(net.requests, [])
+
+    def test_an_omitted_locality_searches_regionally_on_purpose(self):
+        gate = FakeGate()
+        net = FakeNet(gate, *ANSWERS)
+        code, lines, text = self.search(net, gate)
+        self.assertEqual(code, 0, text)
+        self.assertEqual(len(net.requests), 4)
+        for req in net.requests:
+            self.assertNotIn('l=', req.full_url)
+
+    def test_an_unknown_locality_stops_before_any_search_request(self):
+        gate = FakeGate()
+        net = FakeNet(gate, (200, LOCALITIES))
+        code, _, text = self.search(net, gate, ('--locality', 'Noloconozco'))
+        self.assertEqual(code, 64)
+        self.assertIn('not a listed locality', text)
+        self.assertEqual(len(net.requests), 1)   # only the form; no block was asked for
+
     def test_search_reads_the_four_blocks_and_writes_candidates_gwt_rank_ranks(self):
         gate = FakeGate()
         net = FakeNet(gate, *ANSWERS)