From b120c39f3fa9968fbbaf2ee224d4d7a97e8e4724 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 7 Nov 2026 03:32:44 +0100 Subject: [PATCH 1/2] fix(ledger): stop a logger with no ledger named from writing into a stray ./buscado.tsv --- CHANGELOG.md | 1 + docs/ledger.md | 2 +- ledger/gwt_ledger.py | 19 +++++++++++++++++- ledger/test_gwt_ledger.py | 41 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4475384..96835dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Every release has a **Change lists** heading. It names each change that makes a - `gwt-read genae search --locality` now checks the value against the site's search form for the province: a listed name becomes its id, and an id is confirmed to belong to that province. Before, a name or an unknown id went out as `l=` and the site silently ignored it, so the search looked filtered but returned the whole province. An unknown, ambiguous or blank value now exits 64 (`--locality ''` used to search the whole province), and the run prints which locality it searched (#246). - `gwt-read pares search` no longer pauses every lane when one query gets an empty or unreadable page. It reads the source's canary once, live, through the gate; when the canary answers with at least its minimum, the search exits 7 (`unreadable`, new), saves and caches nothing for that query, and the gate only logs it. A canary that is empty, short, walled, unreachable or the search itself keeps the wall (exit 3). Record an exit 7 as `unproven`, never `none` (#237). +- `gwt-ledger add` and `import-tsv` no longer create `./buscado.tsv` when no ledger is named. With neither `--ledger` nor `$GWT_LEDGER`, and no `./buscado.tsv` in the current folder, they exit 2, name the path and write nothing. Before, a logger run without its variables wrote its rows into a new `buscado.tsv` wherever it ran, outside the lane files, and its rerun was then refused as already logged. A `./buscado.tsv` that already exists is still used, as before. ## [1.3.2] - 2026-11-07 diff --git a/docs/ledger.md b/docs/ledger.md index 3f1bae6..4efe00d 100644 --- a/docs/ledger.md +++ b/docs/ledger.md @@ -21,7 +21,7 @@ One row per search, tab-separated, with a header line: | `next_date` | optional ISO date to look again; for an appeal, a few days before the deadline, so there is time to act | | `control` | the proof a `none` stands on: the `--control-ok` reference ([below](#a-none-needs-proof)) | -The path comes from `--ledger`, else `$GWT_LEDGER`, else `./buscado.tsv`. The agent name defaults to `$GWT_AGENT`. +The path comes from `--ledger`, else `$GWT_LEDGER`, else `./buscado.tsv`. That last default is only read, never created: when no ledger is named and `./buscado.tsv` does not exist, `add` and `import-tsv` exit 2, name the path and write nothing, so a logger run from the wrong folder cannot log into a stray file. The agent name defaults to `$GWT_AGENT`. ## Commands diff --git a/ledger/gwt_ledger.py b/ledger/gwt_ledger.py index d582159..0e02fb2 100755 --- a/ledger/gwt_ledger.py +++ b/ledger/gwt_ledger.py @@ -28,6 +28,8 @@ The ledger path comes from --ledger, else $GWT_LEDGER, else ./buscado.tsv. With --ledger-dir (or $GWT_LEDGER_DIR), `add` writes to /.tsv, one file per lane, and every reader reads the ledger plus all /*.tsv. +A write never creates ./buscado.tsv on its own: with neither --ledger nor $GWT_LEDGER, add and import-tsv +stop (exit 2) when that file does not exist yet, so a logger run from the wrong folder cannot log into a stray file. Standard library only (`coverage` also uses the toolkit's gwt_core). """ import argparse, collections, datetime, decimal, difflib, fcntl, glob, gzip, hashlib, json, os, re, sys, unicodedata @@ -727,7 +729,7 @@ def main(argv=None): sys.stderr.write('gwt-ledger: %s\n' % exc) return 2 p = argparse.ArgumentParser(prog='gwt-ledger', description=__doc__.split('\n')[0]) - p.add_argument('--ledger', default=os.environ.get('GWT_LEDGER', 'buscado.tsv')) + p.add_argument('--ledger', default=os.environ.get('GWT_LEDGER') or None) p.add_argument('--ledger-dir', default=os.environ.get('GWT_LEDGER_DIR', ''), help='one file per lane: add writes /.tsv, readers read the ledger and every file here') p.add_argument('--registry', default=os.environ.get('GWT_LEDGER_SOURCES', ''), @@ -819,6 +821,17 @@ def err(msg): sys.stderr.write('gwt-ledger: %s\n' % msg) return 2 + # Nobody named a ledger: reads fall back to ./buscado.tsv, but a write must not create it. A logger run + # without its variables put a lane's rows in a stray ./buscado.tsv, and the rerun was refused as logged. + unnamed = args.ledger is None + if unnamed: + args.ledger = 'buscado.tsv' + + def stray(path): + if unnamed and path == args.ledger and not os.path.exists(path): + return err('no ledger named and %s does not exist: pass --ledger or set $GWT_LEDGER (and --ledger-dir ' + 'or $GWT_LEDGER_DIR for lane files); nothing written' % os.path.abspath(path)) + # A query or person that normalizes to nothing (only punctuation, say) would match every row. for name in ('query', 'person', 'reason'): value = getattr(args, name, None) @@ -855,6 +868,8 @@ def lane_path(): if args.cmd == 'add': path = lane_path() + if stray(path): + return 2 source = source_id(args.source, note=True) if args.control_ok: problem = control_problem(args.control_ok) @@ -883,6 +898,8 @@ def twins(): # a logger run twice must not log every search twice; a voided row 'already logged as %s\n' % ' '.join(found)) return 0 if args.cmd == 'import-tsv': + if not args.ledger_dir and stray(args.ledger): + return 2 return import_tsv(args, reg, err) if args.cmd in ('request', 'costs'): try: diff --git a/ledger/test_gwt_ledger.py b/ledger/test_gwt_ledger.py index eb48e7a..6f55fb8 100644 --- a/ledger/test_gwt_ledger.py +++ b/ledger/test_gwt_ledger.py @@ -621,6 +621,47 @@ def test_import_maps_columns_and_synonyms_and_needs_proof(self): [('familysearch', 'hit'), ('gazeta', 'none'), ('bvph', 'unproven')]) self.assertEqual(rows[0]['agent'], 'import') + def test_a_write_with_no_ledger_named_never_creates_one(self): + # A logger run without $GWT_LEDGER put a lane's rows in a stray ./buscado.tsv of the folder it ran from, and + # its rerun was refused as already logged: an add or import-tsv that names no ledger must not create one. + src = os.path.join(self.dir.name, 'old-log.tsv') + write(src, 'who\twhere\twhat\toutcome\n' 'Ana Zorvalo\tgaceta\tedicto 1902\tnegative\n') + add = ('add', '--person', 'Ana Zorvalo', '--source', 'gaceta', '--query', 'edicto 1902', '--result', 'hit', + '--agent', 'lane-1') + imp = ('import-tsv', src, '--map', 'person_key=who,source=where,query=what,result=outcome') + cwd, env = os.getcwd(), os.environ.pop('GWT_LEDGER', None) + work = os.path.join(self.dir.name, 'elsewhere') + os.makedirs(work) + os.chdir(work) + try: + for argv in (add, imp): + err = io.StringIO() + with redirect_stderr(err): + code, _ = run(*argv) + self.assertEqual(code, 2, argv[0]) + self.assertFalse(os.path.exists('buscado.tsv'), argv[0]) + self.assertIn(os.path.join(os.path.realpath(work), 'buscado.tsv') + ' does not exist', err.getvalue()) + self.assertIn('nothing written', err.getvalue()) + # a ./buscado.tsv that already exists is still the default ledger, as before + write('buscado.tsv', '\t'.join(L.COLUMNS) + '\n') + self.assertEqual(run(*add)[0], 0) + self.assertEqual(len(L.read_rows('buscado.tsv')), 1) + finally: + os.chdir(cwd) + if env is not None: + os.environ['GWT_LEDGER'] = env + # naming the ledger, by flag or by $GWT_LEDGER, still creates it + named = os.path.join(work, 'named.tsv') + self.assertEqual(run('--ledger', named, *add)[0], 0) + os.environ['GWT_LEDGER'] = os.path.join(work, 'by-env.tsv') + try: + self.assertEqual(run(*add)[0], 0) + finally: + os.environ.pop('GWT_LEDGER') + if env is not None: + os.environ['GWT_LEDGER'] = env + self.assertTrue(os.path.exists(named) and os.path.exists(os.path.join(work, 'by-env.tsv'))) + def test_import_checks_each_control_like_add(self): bad = os.path.join(self.dir.name, 'bad.txt') write(bad, 'control\tcontrol\t--name x\t0 read, want at least 3\tSHORT\n') From 068381b6fc35bae6eab139fb3897c7f292a7f0c8 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 7 Nov 2026 03:43:53 +0100 Subject: [PATCH 2/2] fix(ledger): say a ledger directory still writes lane files, and keep the test off a lane's variables --- CHANGELOG.md | 2 +- docs/ledger.md | 2 +- ledger/gwt_ledger.py | 5 ++-- ledger/test_gwt_ledger.py | 51 ++++++++++++++++++--------------------- 4 files changed, 28 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96835dc..b6320e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Every release has a **Change lists** heading. It names each change that makes a - `gwt-read genae search --locality` now checks the value against the site's search form for the province: a listed name becomes its id, and an id is confirmed to belong to that province. Before, a name or an unknown id went out as `l=` and the site silently ignored it, so the search looked filtered but returned the whole province. An unknown, ambiguous or blank value now exits 64 (`--locality ''` used to search the whole province), and the run prints which locality it searched (#246). - `gwt-read pares search` no longer pauses every lane when one query gets an empty or unreadable page. It reads the source's canary once, live, through the gate; when the canary answers with at least its minimum, the search exits 7 (`unreadable`, new), saves and caches nothing for that query, and the gate only logs it. A canary that is empty, short, walled, unreachable or the search itself keeps the wall (exit 3). Record an exit 7 as `unproven`, never `none` (#237). -- `gwt-ledger add` and `import-tsv` no longer create `./buscado.tsv` when no ledger is named. With neither `--ledger` nor `$GWT_LEDGER`, and no `./buscado.tsv` in the current folder, they exit 2, name the path and write nothing. Before, a logger run without its variables wrote its rows into a new `buscado.tsv` wherever it ran, outside the lane files, and its rerun was then refused as already logged. A `./buscado.tsv` that already exists is still used, as before. +- `gwt-ledger add` and `import-tsv` no longer create `./buscado.tsv` when no ledger is named. With neither `--ledger` nor `$GWT_LEDGER`, no `--ledger-dir` or `$GWT_LEDGER_DIR`, and no `./buscado.tsv` in the current folder, they exit 2, name the path and write nothing. Before, a logger run without its variables wrote its rows into a new `buscado.tsv` wherever it ran, outside the lane files, and its rerun was then refused as already logged. A `./buscado.tsv` that already exists is still used, and lane files under a ledger directory are written as before. ## [1.3.2] - 2026-11-07 diff --git a/docs/ledger.md b/docs/ledger.md index 4efe00d..ca22053 100644 --- a/docs/ledger.md +++ b/docs/ledger.md @@ -21,7 +21,7 @@ One row per search, tab-separated, with a header line: | `next_date` | optional ISO date to look again; for an appeal, a few days before the deadline, so there is time to act | | `control` | the proof a `none` stands on: the `--control-ok` reference ([below](#a-none-needs-proof)) | -The path comes from `--ledger`, else `$GWT_LEDGER`, else `./buscado.tsv`. That last default is only read, never created: when no ledger is named and `./buscado.tsv` does not exist, `add` and `import-tsv` exit 2, name the path and write nothing, so a logger run from the wrong folder cannot log into a stray file. The agent name defaults to `$GWT_AGENT`. +The path comes from `--ledger`, else `$GWT_LEDGER`, else `./buscado.tsv`. A write never creates that last default: when no ledger is named, no `--ledger-dir` or `$GWT_LEDGER_DIR` is set and `./buscado.tsv` does not exist, `add` and `import-tsv` exit 2, name the path and write nothing, so a logger run from the wrong folder cannot log into a stray file. An existing `./buscado.tsv` is still written, and with a ledger directory both commands write the agent's lane file as before. The agent name defaults to `$GWT_AGENT`. ## Commands diff --git a/ledger/gwt_ledger.py b/ledger/gwt_ledger.py index 0e02fb2..2b70644 100755 --- a/ledger/gwt_ledger.py +++ b/ledger/gwt_ledger.py @@ -28,8 +28,9 @@ The ledger path comes from --ledger, else $GWT_LEDGER, else ./buscado.tsv. With --ledger-dir (or $GWT_LEDGER_DIR), `add` writes to /.tsv, one file per lane, and every reader reads the ledger plus all /*.tsv. -A write never creates ./buscado.tsv on its own: with neither --ledger nor $GWT_LEDGER, add and import-tsv -stop (exit 2) when that file does not exist yet, so a logger run from the wrong folder cannot log into a stray file. +A write never creates ./buscado.tsv on its own: with neither --ledger nor $GWT_LEDGER, and no --ledger-dir, add and +import-tsv stop (exit 2) when that file does not exist yet, so a logger run from the wrong folder cannot log into a +stray file. An existing ./buscado.tsv is still written; with --ledger-dir they write the lane file as before. Standard library only (`coverage` also uses the toolkit's gwt_core). """ import argparse, collections, datetime, decimal, difflib, fcntl, glob, gzip, hashlib, json, os, re, sys, unicodedata diff --git a/ledger/test_gwt_ledger.py b/ledger/test_gwt_ledger.py index 6f55fb8..eb05a5f 100644 --- a/ledger/test_gwt_ledger.py +++ b/ledger/test_gwt_ledger.py @@ -629,37 +629,32 @@ def test_a_write_with_no_ledger_named_never_creates_one(self): add = ('add', '--person', 'Ana Zorvalo', '--source', 'gaceta', '--query', 'edicto 1902', '--result', 'hit', '--agent', 'lane-1') imp = ('import-tsv', src, '--map', 'person_key=who,source=where,query=what,result=outcome') - cwd, env = os.getcwd(), os.environ.pop('GWT_LEDGER', None) work = os.path.join(self.dir.name, 'elsewhere') os.makedirs(work) - os.chdir(work) - try: - for argv in (add, imp): - err = io.StringIO() - with redirect_stderr(err): - code, _ = run(*argv) - self.assertEqual(code, 2, argv[0]) - self.assertFalse(os.path.exists('buscado.tsv'), argv[0]) - self.assertIn(os.path.join(os.path.realpath(work), 'buscado.tsv') + ' does not exist', err.getvalue()) - self.assertIn('nothing written', err.getvalue()) - # a ./buscado.tsv that already exists is still the default ledger, as before - write('buscado.tsv', '\t'.join(L.COLUMNS) + '\n') - self.assertEqual(run(*add)[0], 0) - self.assertEqual(len(L.read_rows('buscado.tsv')), 1) - finally: - os.chdir(cwd) - if env is not None: - os.environ['GWT_LEDGER'] = env - # naming the ledger, by flag or by $GWT_LEDGER, still creates it - named = os.path.join(work, 'named.tsv') - self.assertEqual(run('--ledger', named, *add)[0], 0) - os.environ['GWT_LEDGER'] = os.path.join(work, 'by-env.tsv') - try: + cwd = os.getcwd() + env = {k: v for k, v in os.environ.items() if k not in ('GWT_LEDGER', 'GWT_LEDGER_DIR')} + with mock.patch.dict(os.environ, env, clear=True): # a lane's own variables must not leak in + os.chdir(work) + try: + for argv in (add, imp): + err = io.StringIO() + with redirect_stderr(err): + code, _ = run(*argv) + self.assertEqual(code, 2, argv[0]) + self.assertFalse(os.path.exists('buscado.tsv'), argv[0]) + self.assertIn(os.path.join(os.path.realpath(work), 'buscado.tsv') + ' does not exist', err.getvalue()) + self.assertIn('nothing written', err.getvalue()) + # a ./buscado.tsv that already exists is still the default ledger, as before + write('buscado.tsv', '\t'.join(L.COLUMNS) + '\n') + self.assertEqual(run(*add)[0], 0) + self.assertEqual(len(L.read_rows('buscado.tsv')), 1) + finally: + os.chdir(cwd) + # naming the ledger, by flag or by $GWT_LEDGER, still creates it + named = os.path.join(work, 'named.tsv') + self.assertEqual(run('--ledger', named, *add)[0], 0) + os.environ['GWT_LEDGER'] = os.path.join(work, 'by-env.tsv') self.assertEqual(run(*add)[0], 0) - finally: - os.environ.pop('GWT_LEDGER') - if env is not None: - os.environ['GWT_LEDGER'] = env self.assertTrue(os.path.exists(named) and os.path.exists(os.path.join(work, 'by-env.tsv'))) def test_import_checks_each_control_like_add(self):