From 75027a9452c470bffeb6b46721db26e1bfcdcfec Mon Sep 17 00:00:00 2001 From: Instinct Agent Date: Fri, 6 Nov 2026 22:19:56 +0000 Subject: [PATCH 1/2] fix(read): distinguish unreadable PARES query responses --- docs/read.md | 7 ++- read/gwt_read.py | 34 +++++++++-- read/gwt_reader.py | 19 +++++- read/mutants.d/pares-unreadable.txt | 2 + read/test_gwt_pares.py | 90 ++++++++++++++++++++++++++++- 5 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 read/mutants.d/pares-unreadable.txt diff --git a/docs/read.md b/docs/read.md index ad1046bb..cf78cdb6 100644 --- a/docs/read.md +++ b/docs/read.md @@ -44,7 +44,8 @@ Every request goes the same way (the INSEE source makes none). A fresh answer in | HTTP 2xx with the JSON it expects, or a page with the record text it asked for | writes the envelope and the candidates | 0, or 2 with no results | | HTTP 403 | blocks the account's gate for 24 h | 3 | | HTTP 429, a captcha or block page | pauses every lane (20 min) and triples the gap | 3 | -| HTTP 200 with an empty body, a page that is not JSON from a JSON site, or a page without the record text from a page site (an empty frame, an Imperva block inside an iframe, the site's own error page) | read as a block: pauses every lane | 3 | +| HTTP 200 with an empty body, a page that is not JSON from a JSON site, or a page without the record text from a page site (an empty frame, an Imperva block inside an iframe, the site's own error page) | read as a block: pauses every lane (on PARES, only once the source's canary has failed to answer too) | 3 | +| an empty or unreadable results page on PARES whose canary answers (checked once, live, at the gate's pace) | the query's own problem, not a block: the unreadable query is neither saved nor cached (the successful form and canary answers may be), and the gate hears nothing; log the row as unreadable and carry on with the batch | 7 | | HTTP 401, or a sign-in page | pauses every lane; get a fresh token | 4 | | HTTP 403 for one FamilySearch record whose access is restricted (see [restricted records](#a-403-for-one-record-restricted-or-blocked)) | writes the envelope with `access: restricted` and no candidates; the gate only logs it as a denial (no pause, no block) | 5 | | JSON of an unexpected shape, or another HTTP error | nothing saved, the gate is not told | 1 | @@ -65,6 +66,8 @@ gwt read boe search --name "Ana Zorvalo Quemada" --controls both --out runs/ # control control --given Cristóbal --surname Colón 25 read, want at least 10 ok ``` +A PARES search that gets an empty page checks it once against the canary before it counts as a wall, at the gate's normal spacing. When the canary answers, the problem is that query, not the site: the search exits 7 (`unreadable`), the unreadable query is neither saved nor cached (the successful form and canary answers may be) and the gate hears nothing, so one odd query no longer pauses every lane or voids its batch. When the canary is empty too, or reads short of its minimum, the wall stands as before; a canary that answered short may be cached, and the probe always reads it live. + `--controls before|after|both` on `search` runs the canaries around that search, and reads the search itself without the cache, since the canaries only prove the site works now. A short canary before the search means no search. A short canary after it, or one stopped by a block or a captcha, makes the run void (exit 3): the envelope and candidates files the search wrote are removed (one `removed` line each), and so are the answers it cached, so nothing on disk reads as "none" and the next run asks the site again. To bracket a batch, run `control` before the first search and after the last one: a canary costs one search, which is two requests on PARES (the search form, then the search), so do not add it to every search. | Source | Canary | Minimum rows | @@ -331,4 +334,4 @@ cd read && python3 -m unittest -v test_gwt_read test_gwt_pares test_gwt_genae te `test_registry` checks every source against the registry's shape, the three-state page classifier on fixture pages (a skeleton page is `unknown`, a STOP), and that a restricted FamilySearch record never reaches the gate while a refused control record still blocks it. `test_controls` checks that a canary skips the cache, that a short one voids the run, and that `--controls` brackets a search. -Invented people, descriptions and ids (the PARES pages in `read/fixtures/` keep the site's markup with invented content), a fake transport and a fake gate: the tests make no request to any site. They check that a search writes an envelope and candidates `gwt-rank` ranks, that a repeated search is served from the cache with no network call, that the gate is asked before every request, that a 403, 429, captcha, sign-in page or empty frame stops the run with nothing written or cached, and that the token never appears in a written file or a log line. The INSEE tests use invented fixed-width lines (`read/fixtures/insee-deces-1985.txt`, and lines built in the test). They check that the source asks for no gate, cache or network, that each line out of the layout exits 2 with nothing written, and that a value INSEE wrote out of its own rules is kept and marked. +Invented people, descriptions and ids (the PARES pages in `read/fixtures/` keep the site's markup with invented content), a fake transport and a fake gate: the tests make no request to any site. They check that a search writes an envelope and candidates `gwt-rank` ranks, that a repeated search is served from the cache with no network call, that the gate is asked before every request, that a 403, 429, captcha, sign-in page or empty frame stops the run with nothing written or cached, that an empty PARES page its canary answers is unreadable (exit 7, no gate hit) and stays a wall when the canary cannot answer either, and that the token never appears in a written file or a log line. The INSEE tests use invented fixed-width lines (`read/fixtures/insee-deces-1985.txt`, and lines built in the test). They check that the source asks for no gate, cache or network, that each line out of the layout exits 2 with nothing written, and that a value INSEE wrote out of its own rules is kept and marked. diff --git a/read/gwt_read.py b/read/gwt_read.py index bf926b2c..66da39fc 100644 --- a/read/gwt_read.py +++ b/read/gwt_read.py @@ -35,7 +35,9 @@ (HTTP 429: every lane on the account pauses; HTTP 403: the account's gate is blocked for 24 h; do other work; also a canary that read less than its minimum: the run is void), 4 signed out or the token expired (get a fresh token; never retry in a loop), 5 restricted (a FamilySearch record refused to this access -level, not a block; the gate is not told), 6 the account's gate is paused (nothing was requested), 64 usage (no token). +level, not a block; the gate is not told), 6 the account's gate is paused (nothing was requested), 7 unreadable +(pares: an empty page the source's control answered: this query's own problem, not a block; nothing saved, the +gate is not told), 64 usage (no token). """ import argparse, contextlib, io, json, os, re, shutil, sys, tempfile, urllib.error, urllib.parse, urllib.request @@ -326,6 +328,23 @@ def cmd_record(a, R): return gwt_reader.EXIT_RESTRICTED +def _pares_control_probe(R, h): + """The on_empty check for a pares search: the source's declared canary, read live through the gate at + its normal pace. True only when the canary answers with at least its minimum: the site is fine, so the empty + page is that query's own problem. A canary that reads short keeps the wall; one that cannot answer raises its + own Stop from inside the check, with its own gate signal.""" + ctl = next(s['controls'][0] for s in LEGACY if s['id'] == 'pares') + args = argparse.ArgumentParser() + _pares_search_args(args) + ns = args.parse_args(list(ctl['argv'])) + url = gwt_pares.search_url(ns.given, ns.surname, ns.words) + + def on_empty(): + _st, _rh, body, _cached = R.get(url, h, refresh=True, page=gwt_pares.is_results) + return len(gwt_pares.search_candidates(body.decode('utf-8', 'replace'))) >= ctl['min_count'] + return on_empty + + def cmd_pares_search(a, R): if not (a.given or a.surname or a.words): err('give --given, --surname, --words or any of them') @@ -337,7 +356,14 @@ def cmd_pares_search(a, R): if a.refresh or R.cache.lookup('GET', url, h) is None: R.get(gwt_pares.FORM_URL, h, refresh=True, page=gwt_pares.is_form) key = gwt_reader.search_key(url, ['pares', a.given] + list(a.surname) + [a.words]) - return _save_page(a, R, url, h, gwt_pares.is_results, key, gwt_pares.search_candidates, gwt_pares.total) + try: + return _save_page(a, R, url, h, gwt_pares.is_results, key, gwt_pares.search_candidates, gwt_pares.total, + on_empty=_pares_control_probe(R, h)) + except gwt_reader.Unreadable as e: + err(str(e)) + out("results\tunreadable: the page came back empty, but the source's control answered; this query, " + "not the site (nothing saved, no gate hit)") + return gwt_reader.EXIT_UNREADABLE def cmd_pares_record(a, R): @@ -495,8 +521,8 @@ def cmd_insee_search(a, R=None): return EXIT_OK if cands else EXIT_EMPTY -def _save_page(a, R, url, h, page_ok, key, to_cands, total_of): - status, rh, body, cached = R.get(url, h, refresh=a.refresh, page=page_ok) +def _save_page(a, R, url, h, page_ok, key, to_cands, total_of, on_empty=None): + status, rh, body, cached = R.get(url, h, refresh=a.refresh, page=page_ok, on_empty=on_empty) page = body.decode('utf-8', 'replace') cands = to_cands(page) env = gwt_reader.envelope(url, (rh or {}).get(FINAL_URL_HEADER), text=gwt_pares.text(page), html=page, diff --git a/read/gwt_reader.py b/read/gwt_reader.py index a7405a12..9cd7bd68 100644 --- a/read/gwt_reader.py +++ b/read/gwt_reader.py @@ -24,6 +24,7 @@ import datetime, hashlib, json, os, re, struct, unicodedata EXIT_OK, EXIT_ERROR, EXIT_EMPTY, EXIT_WALL, EXIT_DEAD, EXIT_PAUSED = 0, 1, 2, 3, 4, 6 +EXIT_UNREADABLE = 7 # a query-specific unreadable answer whose live control succeeds EXIT_RESTRICTED = 5 # the site refuses this one record to every account of this access level: a result, not a block # ---------------------------------------------------------------- small helpers @@ -280,6 +281,13 @@ def __init__(self, msg, status=403, body=b''): self.status, self.body = status, body +class Unreadable(Stop): + """An unreadable query with a successful live control: a result, not a gate hit.""" + + def __init__(self, msg): + super().__init__(msg, EXIT_UNREADABLE, verdict='unreadable') + + class Reader: """GET through gwt-fscache and gwt-gate. fetcher(method, url, headers, data) -> (status, headers, body) is gwt-fscache's fetcher signature; the token belongs inside it, never in `headers`, which the cache sees.""" @@ -290,12 +298,15 @@ def __init__(self, cache, gate, lane, fetcher, log=None): self.cache, self.gate, self.lane, self.fetcher = cache, gate, lane, fetcher self.log = log or (lambda m: None) - def get(self, url, headers=None, expect=None, refresh=False, page=None, on_403=None, data=None): + def get(self, url, headers=None, expect=None, refresh=False, page=None, on_403=None, data=None, on_empty=None): """(status, headers, body, from_cache) for a 2xx answer the site really gave; raises Stop otherwise. `expect` checks a JSON answer, `page` an HTML one (see classify_answer). `on_403(body)` is asked about a 403 before the gate blocks: 'restricted' only logs it as a denial (gate.denied: no pause, no block) and raises Restricted; anything else is a block, as without it. `data` (bytes) makes it a POST, for a site whose search - is a form or a JSON body; the cache keeps a POST only when it was built with methods=('GET', 'POST').""" + is a form or a JSON body; the cache keeps a POST only when it was built with methods=('GET', 'POST'). + `on_empty()` is checked before an empty response hits the gate. A successful live source control + returns true and raises Unreadable with no gate hit; false or absent preserves the existing wall. + A failed control raises its own Stop and records its own gate signal.""" state = {} def gated(method, u, h, data): @@ -314,6 +325,10 @@ def gated(method, u, h, data): raise Restricted('restricted: HTTP 403 at %s is about this record\'s access, not a block; no gate hit' % url, state['status'], state['body']) if verdict != 'ok': + if verdict == 'empty' and on_empty is not None and on_empty(): + raise Unreadable(f"empty or unreadable page, but the source's control answered: this query's own " + f"problem, not a block; nothing saved, no gate hit (HTTP {state['status']} " + f"at {url[:120]})") code, why, tell = STOP_FOR[verdict] what = 'HTTP %s %s' % (state['status'], url[:120]) if tell == 'block': diff --git a/read/mutants.d/pares-unreadable.txt b/read/mutants.d/pares-unreadable.txt new file mode 100644 index 00000000..30ecb8d1 --- /dev/null +++ b/read/mutants.d/pares-unreadable.txt @@ -0,0 +1,2 @@ +gwt_reader.py | if verdict == 'empty' and on_empty is not None and on_empty(): | if False: | an empty pares page its own control answers is unreadable (exit 7, no gate hit), never a wall +gwt_read.py | return len(gwt_pares.search_candidates(body.decode('utf-8', 'replace'))) >= ctl['min_count'] | return True | a control that reads fewer rows than its minimum cannot clear an empty page: the wall stands diff --git a/read/test_gwt_pares.py b/read/test_gwt_pares.py index fd9aabde..d389b392 100644 --- a/read/test_gwt_pares.py +++ b/read/test_gwt_pares.py @@ -15,6 +15,15 @@ ERROR_PAGE = (b'
Ha ocurrido un error en la aplicaci\xc3\xb3n. ' b'Por favor, vuelva a intentarlo de nuevo.
') EMPTY_FRAME = b'' +SEARCH_URL = gwt_pares.search_url('Ana', ['Zorvalo'], '') +CONTROL_URL = gwt_pares.search_url('Cristóbal', ['Colón'], '') # the source's declared canary (LEGACY) + + +def control_page(rows): + """A results page with `rows` invented descriptions: the canary's answer when the site is fine.""" + items = b''.join(b'

' + b'Expediente de prueba %d' % (i, i) for i in range(rows)) + return b'

' + items + b'
' class Pares(Base): @@ -127,11 +136,59 @@ def test_incapsula_block_is_a_stop(self): self.assert_stop([(200, FORM), (200, b'
x
')], gwt_reader.EXIT_WALL, 'hits', 2) - def test_empty_frame_is_a_stop(self): - self.assert_stop([(200, FORM), (200, EMPTY_FRAME)], gwt_reader.EXIT_WALL, 'hits', 2) + def test_empty_frame_the_control_answers_is_unreadable_and_the_gate_hears_nothing(self): + gate = FakeGate() + answers = [(200, FORM), (200, EMPTY_FRAME), (200, control_page(15))] + net = FakeNet(gate, *answers * 2) + code, lines, text = self.search(net, gate) + self.assertEqual(code, gwt_reader.EXIT_UNREADABLE, text) + self.assertEqual(gate.hits, [], 'an empty page the control answers is not a block') + self.assertEqual(self.out_files(), [], 'an unreadable page wrote files') + self.assertNotIn('candidates', lines) + self.assertIn('unreadable', lines['results']) + self.assertEqual([r.full_url for r in net.requests], [gwt_pares.FORM_URL, SEARCH_URL, CONTROL_URL]) + self.assertEqual(self.search(net, gate)[0], gwt_reader.EXIT_UNREADABLE) + self.assertEqual(len(net.requests), 6, 'an unreadable answer was served from the cache') + + def test_empty_frame_is_a_stop_when_the_control_is_empty_too(self): + gate = FakeGate() + answers = [(200, FORM), (200, EMPTY_FRAME), (200, EMPTY_FRAME)] + net = FakeNet(gate, *answers * 2) + code, lines, text = self.search(net, gate) + self.assertEqual(code, gwt_reader.EXIT_WALL, text) + self.assertNotIn('candidates', lines) + self.assertEqual(self.out_files(), [], 'a STOP wrote files') + self.assertEqual(len(gate.hits), 1, 'the gate did not hear about it') + self.assertEqual([r.full_url for r in net.requests], [gwt_pares.FORM_URL, SEARCH_URL, CONTROL_URL]) + self.search(net, gate) + self.assertEqual(len(net.requests), 6, 'a STOP answer was served from the cache') + + def test_empty_frame_with_a_short_control_is_still_a_wall(self): + gate = FakeGate() + net = FakeNet(gate, (200, FORM), (200, EMPTY_FRAME), (200, control_page(3))) + code, _, text = self.search(net, gate) + self.assertEqual(code, gwt_reader.EXIT_WALL, text) + self.assertEqual(len(gate.hits), 1, 'a canary under its minimum cannot clear the empty page') + self.assertEqual(self.out_files(), []) + + def test_empty_frame_with_a_captcha_on_the_control_is_the_controls_wall(self): + gate = FakeGate() + net = FakeNet(gate, (200, FORM), (200, EMPTY_FRAME), (200, CAPTCHA)) + code, _, text = self.search(net, gate) + self.assertEqual(code, gwt_reader.EXIT_WALL, text) + self.assertEqual(len(gate.hits), 1, 'the control met the wall first and reported it') + self.assertEqual(self.out_files(), []) def test_site_error_page_is_a_stop(self): - self.assert_stop([(200, FORM), (200, ERROR_PAGE)], gwt_reader.EXIT_WALL, 'hits', 2) + gate = FakeGate() + answers = [(200, FORM), (200, ERROR_PAGE), (200, ERROR_PAGE)] + net = FakeNet(gate, *answers * 2) + code, _, text = self.search(net, gate) + self.assertEqual(code, gwt_reader.EXIT_WALL, text) + self.assertEqual(self.out_files(), []) + self.assertEqual(len(gate.hits), 1) + self.search(net, gate) + self.assertEqual(len(net.requests), 6, 'a STOP answer was served from the cache') def test_403_blocks_the_gate_for_a_day(self): gate = FakeGate() @@ -175,5 +232,32 @@ def test_bad_id_is_refused_without_a_request(self): self.assertEqual(net.requests, []) +class SharedReaderPostCompatibility(unittest.TestCase): + def test_post_data_keeps_its_positional_slot_and_does_not_run_empty_control(self): + class Cache: + def fetch(cache, method, url, headers, data, fetcher, refresh=False): + self.assertEqual((method, data), ('POST', b'query=test')) + return fetcher(method, url, headers, data) + gate = FakeGate() + calls = [] + reader = gwt_reader.Reader(Cache(), gate, 't', lambda *args: (200, {}, b'{"ok":true}')) + answer = reader.get('https://example.invalid/search', None, None, False, None, None, b'query=test', + on_empty=lambda: calls.append('unexpected')) + self.assertEqual(answer, (200, {}, b'{"ok":true}', False)) + self.assertEqual(calls, []) + + def test_empty_post_with_successful_control_is_unreadable_without_gate_hit(self): + class Cache: + def fetch(cache, method, url, headers, data, fetcher, refresh=False): + self.assertEqual((method, data), ('POST', b'query=test')) + return fetcher(method, url, headers, data) + gate = FakeGate() + reader = gwt_reader.Reader(Cache(), gate, 't', lambda *args: (200, {}, b'')) + with self.assertRaises(gwt_reader.Unreadable) as stopped: + reader.get('https://example.invalid/search', data=b'query=test', on_empty=lambda: True) + self.assertEqual(stopped.exception.exit_code, 7) + self.assertEqual((gate.hits, gate.blocks, gate.deads), ([], [], [])) + + if __name__ == '__main__': unittest.main() From 837b2feb1881a4e189a9a13217cdc16b8191f99b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:04:01 +0100 Subject: [PATCH 2/2] fix(read): keep the wall when the PARES canary cannot prove the site is fine An empty PARES page followed by an unreachable canary (a 5xx, a dropped connection) or a paused gate escaped as exit 1 or 6 with no gate signal, and an empty page on the canary search itself was probed with the same URL and read as unreadable. Both now keep the wall (exit 3, one gate hit). An unreadable answer now leaves a line in the gate log through gate.denied, like a restricted record, so a run of them can be seen. The docs name the ledger result to record (unproven), the mutant rows move to read/mutants.txt with rows for each fix and the minimum-count boundary, the signature-only POST test is cut, and the changelog gets a Fixed entry. --- CHANGELOG.md | 4 ++ docs/read.md | 6 +-- read/gwt_read.py | 26 +++++++---- read/gwt_reader.py | 6 +-- read/mutants.d/pares-unreadable.txt | 2 - read/mutants.txt | 6 +++ read/test_gwt_pares.py | 68 +++++++++++++++++++++-------- 7 files changed, 83 insertions(+), 35 deletions(-) delete mode 100644 read/mutants.d/pares-unreadable.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 707c40cd..e2868748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ Every release has a **Change lists** heading. It names each change that makes a ## [Unreleased] +### Fixed + +- `gwt-read pares search` no longer pauses every lane when one query gets an empty or unreadable page. It reads the source's canary once, live, through the gate; when the canary answers with at least its minimum, the search exits 7 (`unreadable`, new), saves and caches nothing for that query, and the gate only logs it. A canary that is empty, short, walled, unreachable or the search itself keeps the wall (exit 3). Record an exit 7 as `unproven`, never `none` (#237). + ## [1.3.1] - 2026-11-06 ### Change lists diff --git a/docs/read.md b/docs/read.md index cf78cdb6..dde1e846 100644 --- a/docs/read.md +++ b/docs/read.md @@ -45,7 +45,7 @@ Every request goes the same way (the INSEE source makes none). A fresh answer in | HTTP 403 | blocks the account's gate for 24 h | 3 | | HTTP 429, a captcha or block page | pauses every lane (20 min) and triples the gap | 3 | | HTTP 200 with an empty body, a page that is not JSON from a JSON site, or a page without the record text from a page site (an empty frame, an Imperva block inside an iframe, the site's own error page) | read as a block: pauses every lane (on PARES, only once the source's canary has failed to answer too) | 3 | -| an empty or unreadable results page on PARES whose canary answers (checked once, live, at the gate's pace) | the query's own problem, not a block: the unreadable query is neither saved nor cached (the successful form and canary answers may be), and the gate hears nothing; log the row as unreadable and carry on with the batch | 7 | +| an empty or unreadable results page on PARES whose canary answers (checked once, live, at the gate's pace) | the query's own problem, not a block: the unreadable query is neither saved nor cached (the successful form and canary answers may be), and the gate only logs it (no pause); record the row as `unproven` in gwt-ledger, never `none`, and carry on with the batch | 7 | | HTTP 401, or a sign-in page | pauses every lane; get a fresh token | 4 | | HTTP 403 for one FamilySearch record whose access is restricted (see [restricted records](#a-403-for-one-record-restricted-or-blocked)) | writes the envelope with `access: restricted` and no candidates; the gate only logs it as a denial (no pause, no block) | 5 | | JSON of an unexpected shape, or another HTTP error | nothing saved, the gate is not told | 1 | @@ -66,7 +66,7 @@ gwt read boe search --name "Ana Zorvalo Quemada" --controls both --out runs/ # control control --given Cristóbal --surname Colón 25 read, want at least 10 ok ``` -A PARES search that gets an empty page checks it once against the canary before it counts as a wall, at the gate's normal spacing. When the canary answers, the problem is that query, not the site: the search exits 7 (`unreadable`), the unreadable query is neither saved nor cached (the successful form and canary answers may be) and the gate hears nothing, so one odd query no longer pauses every lane or voids its batch. When the canary is empty too, or reads short of its minimum, the wall stands as before; a canary that answered short may be cached, and the probe always reads it live. +A PARES search that gets an empty page checks it once against the canary before it counts as a wall, at the gate's normal spacing. When the canary answers, the problem is that query, not the site: the search exits 7 (`unreadable`), the unreadable query is neither saved nor cached (the successful form and canary answers may be) and the gate only writes an `UNREADABLE` line to its log, so one odd query no longer pauses every lane or voids its batch. Record the row as `unproven` in gwt-ledger (open, to retry or narrow), never `none`. When the canary is empty too, reads short of its minimum, cannot be reached or meets a paused gate, the wall stands as before, and an empty page on the canary search itself is always a wall; a canary that answered short may be cached, and the probe always reads it live. `--controls before|after|both` on `search` runs the canaries around that search, and reads the search itself without the cache, since the canaries only prove the site works now. A short canary before the search means no search. A short canary after it, or one stopped by a block or a captcha, makes the run void (exit 3): the envelope and candidates files the search wrote are removed (one `removed` line each), and so are the answers it cached, so nothing on disk reads as "none" and the next run asks the site again. To bracket a batch, run `control` before the first search and after the last one: a canary costs one search, which is two requests on PARES (the search form, then the search), so do not add it to every search. @@ -334,4 +334,4 @@ cd read && python3 -m unittest -v test_gwt_read test_gwt_pares test_gwt_genae te `test_registry` checks every source against the registry's shape, the three-state page classifier on fixture pages (a skeleton page is `unknown`, a STOP), and that a restricted FamilySearch record never reaches the gate while a refused control record still blocks it. `test_controls` checks that a canary skips the cache, that a short one voids the run, and that `--controls` brackets a search. -Invented people, descriptions and ids (the PARES pages in `read/fixtures/` keep the site's markup with invented content), a fake transport and a fake gate: the tests make no request to any site. They check that a search writes an envelope and candidates `gwt-rank` ranks, that a repeated search is served from the cache with no network call, that the gate is asked before every request, that a 403, 429, captcha, sign-in page or empty frame stops the run with nothing written or cached, that an empty PARES page its canary answers is unreadable (exit 7, no gate hit) and stays a wall when the canary cannot answer either, and that the token never appears in a written file or a log line. The INSEE tests use invented fixed-width lines (`read/fixtures/insee-deces-1985.txt`, and lines built in the test). They check that the source asks for no gate, cache or network, that each line out of the layout exits 2 with nothing written, and that a value INSEE wrote out of its own rules is kept and marked. +Invented people, descriptions and ids (the PARES pages in `read/fixtures/` keep the site's markup with invented content), a fake transport and a fake gate: the tests make no request to any site. They check that a search writes an envelope and candidates `gwt-rank` ranks, that a repeated search is served from the cache with no network call, that the gate is asked before every request, that a 403, 429, captcha, sign-in page or empty frame stops the run with nothing written or cached, that an empty PARES page its canary answers is unreadable (exit 7, only a line in the gate log) and stays a wall when the canary cannot answer either or is the search itself, and that the token never appears in a written file or a log line. The INSEE tests use invented fixed-width lines (`read/fixtures/insee-deces-1985.txt`, and lines built in the test). They check that the source asks for no gate, cache or network, that each line out of the layout exits 2 with nothing written, and that a value INSEE wrote out of its own rules is kept and marked. diff --git a/read/gwt_read.py b/read/gwt_read.py index 66da39fc..4e4dd3b2 100644 --- a/read/gwt_read.py +++ b/read/gwt_read.py @@ -37,7 +37,7 @@ expired (get a fresh token; never retry in a loop), 5 restricted (a FamilySearch record refused to this access level, not a block; the gate is not told), 6 the account's gate is paused (nothing was requested), 7 unreadable (pares: an empty page the source's control answered: this query's own problem, not a block; nothing saved, the -gate is not told), 64 usage (no token). +gate only logs it), 64 usage (no token). """ import argparse, contextlib, io, json, os, re, shutil, sys, tempfile, urllib.error, urllib.parse, urllib.request @@ -328,19 +328,29 @@ def cmd_record(a, R): return gwt_reader.EXIT_RESTRICTED -def _pares_control_probe(R, h): +def _pares_control_probe(R, url, h): """The on_empty check for a pares search: the source's declared canary, read live through the gate at its normal pace. True only when the canary answers with at least its minimum: the site is fine, so the empty - page is that query's own problem. A canary that reads short keeps the wall; one that cannot answer raises its - own Stop from inside the check, with its own gate signal.""" + page is that query's own problem. A canary that reads short, cannot be reached (a 5xx, a dropped connection) + or meets a paused gate keeps the wall; one that meets a wall itself raises that Stop, with its own gate signal. + None when `url` is the canary: a canary never proves itself.""" ctl = next(s['controls'][0] for s in LEGACY if s['id'] == 'pares') args = argparse.ArgumentParser() _pares_search_args(args) ns = args.parse_args(list(ctl['argv'])) - url = gwt_pares.search_url(ns.given, ns.surname, ns.words) + probe = gwt_pares.search_url(ns.given, ns.surname, ns.words) + if url == probe: + return None def on_empty(): - _st, _rh, body, _cached = R.get(url, h, refresh=True, page=gwt_pares.is_results) + try: + _st, _rh, body, _cached = R.get(probe, h, refresh=True, page=gwt_pares.is_results) + except gwt_reader.Stop as e: + if e.verdict in ('http', 'unexpected'): + return False # no proof the site is fine: the empty page keeps its wall + raise + except Paused as _paused: # another lane paused the gate: no proof either + return False return len(gwt_pares.search_candidates(body.decode('utf-8', 'replace'))) >= ctl['min_count'] return on_empty @@ -358,11 +368,11 @@ def cmd_pares_search(a, R): key = gwt_reader.search_key(url, ['pares', a.given] + list(a.surname) + [a.words]) try: return _save_page(a, R, url, h, gwt_pares.is_results, key, gwt_pares.search_candidates, gwt_pares.total, - on_empty=_pares_control_probe(R, h)) + on_empty=_pares_control_probe(R, url, h)) except gwt_reader.Unreadable as e: err(str(e)) out("results\tunreadable: the page came back empty, but the source's control answered; this query, " - "not the site (nothing saved, no gate hit)") + "not the site (nothing saved, no gate hit; log it as unproven)") return gwt_reader.EXIT_UNREADABLE diff --git a/read/gwt_reader.py b/read/gwt_reader.py index 9cd7bd68..0cb90463 100644 --- a/read/gwt_reader.py +++ b/read/gwt_reader.py @@ -304,9 +304,8 @@ def get(self, url, headers=None, expect=None, refresh=False, page=None, on_403=N 403 before the gate blocks: 'restricted' only logs it as a denial (gate.denied: no pause, no block) and raises Restricted; anything else is a block, as without it. `data` (bytes) makes it a POST, for a site whose search is a form or a JSON body; the cache keeps a POST only when it was built with methods=('GET', 'POST'). - `on_empty()` is checked before an empty response hits the gate. A successful live source control - returns true and raises Unreadable with no gate hit; false or absent preserves the existing wall. - A failed control raises its own Stop and records its own gate signal.""" + `on_empty()` is asked about an empty answer before the gate hears it: true (a live source control answered) + only logs it to the gate (gate.denied: no pause) and raises Unreadable; false or absent keeps the wall.""" state = {} def gated(method, u, h, data): @@ -326,6 +325,7 @@ def gated(method, u, h, data): state['status'], state['body']) if verdict != 'ok': if verdict == 'empty' and on_empty is not None and on_empty(): + self.gate.denied(self.lane, 'UNREADABLE HTTP %s %s' % (state['status'], url[:120])) raise Unreadable(f"empty or unreadable page, but the source's control answered: this query's own " f"problem, not a block; nothing saved, no gate hit (HTTP {state['status']} " f"at {url[:120]})") diff --git a/read/mutants.d/pares-unreadable.txt b/read/mutants.d/pares-unreadable.txt deleted file mode 100644 index 30ecb8d1..00000000 --- a/read/mutants.d/pares-unreadable.txt +++ /dev/null @@ -1,2 +0,0 @@ -gwt_reader.py | if verdict == 'empty' and on_empty is not None and on_empty(): | if False: | an empty pares page its own control answers is unreadable (exit 7, no gate hit), never a wall -gwt_read.py | return len(gwt_pares.search_candidates(body.decode('utf-8', 'replace'))) >= ctl['min_count'] | return True | a control that reads fewer rows than its minimum cannot clear an empty page: the wall stands diff --git a/read/mutants.txt b/read/mutants.txt index c23cdf5b..9007206a 100644 --- a/read/mutants.txt +++ b/read/mutants.txt @@ -71,3 +71,9 @@ gwt_read.py | return v if isinstance(v, str) and v else None | return v | place_ gwt_reader.py | if not (isinstance(c, dict) and isinstance(c.get('argv'), list) and c['argv'] | if not (isinstance(c, dict) and isinstance(c.get('argv'), list) | a canary must name at least one search option: an empty search is not a canary gwt_read.py | if not isinstance(t, str) or not t: | if not t: | place_orig keeps only text, never an object a field holds gwt_read.py | R.cache.store = store | pass | a void forgets only the answers its search cached, never the canaries' own +gwt_reader.py | if verdict == 'empty' and on_empty is not None and on_empty(): | if False: | an empty pares page its own control answers is unreadable (exit 7, no gate hit), never a wall +gwt_reader.py | self.gate.denied(self.lane, 'UNREADABLE HTTP %s %s' % (state['status'], url[:120])) | pass | an unreadable pares page leaves a line in the gate log, so a run of them can be seen +gwt_read.py | return len(gwt_pares.search_candidates(body.decode('utf-8', 'replace'))) >= ctl['min_count'] | return True | a control that reads fewer rows than its minimum cannot clear an empty page: the wall stands +gwt_read.py | return len(gwt_pares.search_candidates(body.decode('utf-8', 'replace'))) >= ctl['min_count'] | return len(gwt_pares.search_candidates(body.decode('utf-8', 'replace'))) > ctl['min_count'] | a control that reads exactly its minimum clears an empty page +gwt_read.py | return False # no proof the site is fine: the empty page keeps its wall | return True | a control that cannot be reached (a 5xx, a dropped connection) keeps the empty page's wall +gwt_read.py | if url == probe: | if False: | an empty canary page is a wall: a canary never proves itself diff --git a/read/test_gwt_pares.py b/read/test_gwt_pares.py index d389b392..d0f323ce 100644 --- a/read/test_gwt_pares.py +++ b/read/test_gwt_pares.py @@ -2,7 +2,7 @@ leaves this machine. Run: cd read && python3 -m unittest -v test_gwt_pares""" import contextlib, io, json, os, unittest -from test_gwt_read import Base, FakeGate, FakeNet, CAPTCHA +from test_gwt_read import Base, FakeGate, FakeNet, CAPTCHA, Paused import gwt_pares # noqa: E402 import gwt_read # noqa: E402 import gwt_reader # noqa: E402 @@ -26,6 +26,22 @@ def control_page(rows): return b'
' + items + b'
' +class LoggingGate(FakeGate): + """A fake gate that records the lines only logged to it (gate.denied) and can pause from the n-th request.""" + + def __init__(self, pause_at=None): + super().__init__() + self.denials, self.pause_at = [], pause_at + + def wait(self, lane): + if self.pause_at is not None and len(self.waits) + 1 >= self.pause_at: + raise Paused(600, 'another lane hit the wall') + super().wait(lane) + + def denied(self, lane, what): + self.denials.append(what) + + class Pares(Base): def run_pares(self, args, net, gate): o, e = io.StringIO(), io.StringIO() @@ -136,13 +152,15 @@ def test_incapsula_block_is_a_stop(self): self.assert_stop([(200, FORM), (200, b'
x
')], gwt_reader.EXIT_WALL, 'hits', 2) - def test_empty_frame_the_control_answers_is_unreadable_and_the_gate_hears_nothing(self): - gate = FakeGate() - answers = [(200, FORM), (200, EMPTY_FRAME), (200, control_page(15))] + def test_empty_frame_the_control_answers_is_unreadable_and_the_gate_only_logs_it(self): + gate = LoggingGate() + answers = [(200, FORM), (200, EMPTY_FRAME), (200, control_page(10))] # exactly the canary's minimum net = FakeNet(gate, *answers * 2) code, lines, text = self.search(net, gate) self.assertEqual(code, gwt_reader.EXIT_UNREADABLE, text) self.assertEqual(gate.hits, [], 'an empty page the control answers is not a block') + self.assertEqual(len(gate.denials), 1, 'an unreadable page left no line in the gate log') + self.assertIn('UNREADABLE', gate.denials[0]) self.assertEqual(self.out_files(), [], 'an unreadable page wrote files') self.assertNotIn('candidates', lines) self.assertIn('unreadable', lines['results']) @@ -171,6 +189,31 @@ def test_empty_frame_with_a_short_control_is_still_a_wall(self): self.assertEqual(len(gate.hits), 1, 'a canary under its minimum cannot clear the empty page') self.assertEqual(self.out_files(), []) + def test_empty_frame_with_an_unreachable_control_is_still_a_wall(self): + gate = LoggingGate() + net = FakeNet(gate, (200, FORM), (200, EMPTY_FRAME), (503, b'down')) + code, _, text = self.search(net, gate) + self.assertEqual(code, gwt_reader.EXIT_WALL, text) + self.assertEqual(len(gate.hits), 1, 'a control that cannot answer proves nothing: the empty page is a wall') + self.assertEqual(gate.denials, []) + self.assertEqual(self.out_files(), []) + + def test_empty_frame_whose_control_meets_a_paused_gate_is_still_a_wall(self): + gate = LoggingGate(pause_at=3) + net = FakeNet(gate, (200, FORM), (200, EMPTY_FRAME)) + code, _, text = self.search(net, gate) + self.assertEqual(code, gwt_reader.EXIT_WALL, text) + self.assertEqual(len(gate.hits), 1, 'the empty page never reached the gate') + self.assertEqual(len(net.requests), 2) + + def test_an_empty_canary_page_is_a_wall_and_never_probes_itself(self): + gate = LoggingGate() + net = FakeNet(gate, (200, FORM), (200, EMPTY_FRAME), (200, control_page(15))) + code, _, text = self.run_pares(['search', '--given', 'Cristóbal', '--surname', 'Colón'], net, gate) + self.assertEqual(code, gwt_reader.EXIT_WALL, text) + self.assertEqual(len(gate.hits), 1, 'an empty canary page must pause every lane') + self.assertEqual([r.full_url for r in net.requests], [gwt_pares.FORM_URL, CONTROL_URL]) + def test_empty_frame_with_a_captcha_on_the_control_is_the_controls_wall(self): gate = FakeGate() net = FakeNet(gate, (200, FORM), (200, EMPTY_FRAME), (200, CAPTCHA)) @@ -233,30 +276,17 @@ def test_bad_id_is_refused_without_a_request(self): class SharedReaderPostCompatibility(unittest.TestCase): - def test_post_data_keeps_its_positional_slot_and_does_not_run_empty_control(self): - class Cache: - def fetch(cache, method, url, headers, data, fetcher, refresh=False): - self.assertEqual((method, data), ('POST', b'query=test')) - return fetcher(method, url, headers, data) - gate = FakeGate() - calls = [] - reader = gwt_reader.Reader(Cache(), gate, 't', lambda *args: (200, {}, b'{"ok":true}')) - answer = reader.get('https://example.invalid/search', None, None, False, None, None, b'query=test', - on_empty=lambda: calls.append('unexpected')) - self.assertEqual(answer, (200, {}, b'{"ok":true}', False)) - self.assertEqual(calls, []) - def test_empty_post_with_successful_control_is_unreadable_without_gate_hit(self): class Cache: def fetch(cache, method, url, headers, data, fetcher, refresh=False): self.assertEqual((method, data), ('POST', b'query=test')) return fetcher(method, url, headers, data) - gate = FakeGate() + gate = LoggingGate() reader = gwt_reader.Reader(Cache(), gate, 't', lambda *args: (200, {}, b'')) with self.assertRaises(gwt_reader.Unreadable) as stopped: reader.get('https://example.invalid/search', data=b'query=test', on_empty=lambda: True) self.assertEqual(stopped.exception.exit_code, 7) - self.assertEqual((gate.hits, gate.blocks, gate.deads), ([], [], [])) + self.assertEqual((gate.hits, gate.blocks, gate.deads, len(gate.denials)), ([], [], [], 1)) if __name__ == '__main__':