diff --git a/.claude/skills/translate-page/SKILL.md b/.claude/skills/translate-page/SKILL.md index 763c5c3..af300ad 100644 --- a/.claude/skills/translate-page/SKILL.md +++ b/.claude/skills/translate-page/SKILL.md @@ -11,7 +11,7 @@ exist because each of them was broken once and cost real work. ## Inputs - A page path under `docs/en/`, or a page reported by - `uv run python scripts/translation_status.py` as `missing` or `stale`. + `uv run translation-status` as `missing` or `stale`. - A target language directory, e.g. `docs/fi/`. ## Before translating @@ -79,7 +79,7 @@ The stamp records the git blob hash of the English source the translation was written against. Write it with the helper, never by hand: ```bash -uv run python scripts/stamp_translation.py docs/fi/hardware/index.md +uv run stamp-translation docs/fi/hardware/index.md ``` **Stamp only when you have actually translated.** A stamp updated without real @@ -90,6 +90,12 @@ fixing a typo), the English source did not change: leave the stamp alone. ## Adding a language to the site +`check-glossary` and `check-typography` accept a fixed set of locales, and those +registries live in the `halos-docs-tools` package, not in this repository. A new +locale needs an entry in each, a release of that package, and a bump of the pin +in `pyproject.toml`. Until that lands both commands reject the locale, while +`translation-status` reads `mkdocs.yml` and starts failing the gate immediately. + When a locale is added to `mkdocs.yml`, check the language selector too. The Material theme caps the open menu at `10rem`, which fits five entries at the site's font size; the sixth language onward scrolls out of sight behind a @@ -105,9 +111,9 @@ scrollbar that gives no hint anything is below it. ``` 24rem clears thirteen entries; the viewport term keeps the menu on screen on a -short display. The same block is in the HALPI2 and HALMET repositories — keep -the three identical, and add it to any further site that gains a second -language. +short display. The same block is in the HALPI2, HALMET, SH-RPi and SH-ESP32 +repositories — keep the four identical, and add it to any further site +that gains a second language. Verify by measuring rather than by eye: open the site, read the rule's `max-height` off the stylesheet, and compare it against the list's natural @@ -117,22 +123,22 @@ is captured. ## Verifying -All four, every time: +All five, every time: ```bash uv run mkdocs build --strict -uv run python scripts/check_anchors.py site -uv run python scripts/translation_status.py -uv run python scripts/check_glossary.py fi -uv run python scripts/check_typography.py fi +uv run check-anchors site +uv run translation-status --check +uv run check-glossary fi +uv run check-typography fi ``` **Leave every anchor fragment in its English form while translating**, then map them all at once once the language is complete and the site has been built: ```bash -uv run python scripts/map_anchors.py site fi # report -uv run python scripts/map_anchors.py site fi --apply # rewrite +uv run map-anchors site fi # report +uv run map-anchors site fi --apply # rewrite ``` The mapping is positional — the nth heading of the English page and the nth @@ -144,7 +150,7 @@ the text is in another language. whatever they already say, so the terminology looks consistent right up until a reviewer finds the same connector under two names on adjacent pages. Every language so far shipped that mistake, and each time it landed on the last pages -translated, once the glossary had stopped being opened. `check_glossary.py` +translated, once the glossary had stopped being opened. `check-glossary` reports terms the glossary prescribes and the pages never use — the signature of a rival word having quietly taken over. diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 91aba16..195acaa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 - - run: uv sync + - run: uv sync --locked - run: uv run mkdocs build --strict - uses: actions/upload-pages-artifact@v3 with: diff --git a/.github/workflows/translation-status.yml b/.github/workflows/translation-status.yml index e21dd3b..fdccf5d 100644 --- a/.github/workflows/translation-status.yml +++ b/.github/workflows/translation-status.yml @@ -1,95 +1,32 @@ name: Translation Status +# No paths filter. The gate is a property of the whole repository, not of a +# diff, and a required check that never runs on a pull request touching none of +# the filtered paths leaves that pull request unmergeable forever. on: pull_request: - paths: - - 'docs/**' - - 'mkdocs.yml' - - 'scripts/**' push: branches: [main] workflow_dispatch: +# The called workflow inherits this token, so the comment needs +# pull-requests: write here. Omit it and the run still gates; only the comment +# is skipped. permissions: contents: read pull-requests: write concurrency: group: translation-status-${{ github.ref }} - cancel-in-progress: true + # Pull requests only. On push, github.ref is refs/heads/main for every merge, + # so cancelling lets one merge kill the run checking the one before it -- and + # a cancelled run is grey, not red, so nobody is told. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - status: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # Full history: the report resolves the stamped blob to show the - # English diff since a translation was written. - fetch-depth: 0 - - - uses: astral-sh/setup-uv@v5 - - run: uv sync - - - name: Report translation status - run: | - # tee, not plain redirection: a report only in the job summary is - # invisible in the logs, which is where you look when it misbehaves. - uv run python scripts/translation_status.py --format markdown --diff \ - | tee report.md - cat report.md >> "$GITHUB_STEP_SUMMARY" - - - name: Comment on the pull request - if: github.event_name == 'pull_request' - env: - GH_TOKEN: ${{ github.token }} - PR: ${{ github.event.number }} - run: | - # Only the English pages this PR actually touches. Which paths a PR - # touched is a fact, so a PR editing only translations says nothing. - pages=$(git diff --name-only \ - "origin/${{ github.base_ref }}...HEAD" -- 'docs/en/**/*.md' \ - | sed 's|^docs/en/||') - if [ -z "$pages" ]; then - echo "No English pages touched; nothing to report." - exit 0 - fi - - # shellcheck disable=SC2086 - uv run python scripts/translation_status.py \ - --format markdown --diff --only-pages $pages > comment.md - printf '\n\n' >> comment.md - - existing=$(gh api "repos/${{ github.repository }}/issues/$PR/comments" \ - --jq 'map(select(.body | contains(""))) | .[0].id // empty') - if [ -n "$existing" ]; then - gh api "repos/${{ github.repository }}/issues/comments/$existing" \ - -X PATCH -F body=@comment.md --silent - echo "Updated comment $existing" - else - gh api "repos/${{ github.repository }}/issues/$PR/comments" \ - -F body=@comment.md --silent - echo "Created comment" - fi - - # Last, because unlike a stale translation a broken anchor is actual - # breakage and fails the run — and the report above must still be - # published when it does. - - name: Check anchors - run: | - uv run mkdocs build --strict - # PIPESTATUS, not $?: piping into tee would otherwise mask the - # checker's exit status behind tee's. - set +e - uv run python scripts/check_anchors.py site | tee anchors.txt - broken=${PIPESTATUS[0]} - set -e - { - echo "" - echo "## Anchor check" - echo "" - echo '```' - cat anchors.txt - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - exit "$broken" + # The called workflow builds the site, checks its anchors, and fails the run + # when any translation is stale, missing, unstamped or orphaned. It judges the + # whole repository, not the diff, so an edit to an English page needs its + # translations re-stamped in the same pull request. + translation-status: + uses: halos-org/shared-workflows/.github/workflows/translation-status.yml@main diff --git a/pyproject.toml b/pyproject.toml index 8ed62f8..5a3c4d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,4 +7,5 @@ dependencies = [ "mkdocs-material>=9.5", "click<8.3", "mkdocs-static-i18n>=1.2", + "halos-docs-tools @ git+https://github.com/halos-org/docs-tools@v0.1.0", ] diff --git a/scripts/check_anchors.py b/scripts/check_anchors.py deleted file mode 100644 index 8ad9591..0000000 --- a/scripts/check_anchors.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -"""Verify that every internal anchor in the built site resolves to a real id. - -Anchors are generated from heading text, so translating a heading changes its -slug and silently breaks every link pointing at it — including links on pages -that were not touched, which is why this is a delayed fault: a cross-page anchor -keeps working until its *target* page is translated. `mkdocs build --strict` -does not validate anchors at all. - -Run against a built site directory. Exit status is 1 if any anchor is broken. -""" - -from __future__ import annotations - -import argparse -import os -import re -import sys -from urllib.parse import unquote, urldefrag - -HREF = re.compile(r'href="([^"]+)"') -ID = re.compile(r'\sid="([^"]+)"') - - -def collect_pages(site: str) -> dict[str, set[str]]: - """Map each built page to the set of element ids it defines.""" - ids: dict[str, set[str]] = {} - for root, _, files in os.walk(site): - for name in files: - if name.endswith(".html"): - path = os.path.join(root, name) - text = open(path, encoding="utf-8").read() - ids[os.path.realpath(path)] = set(ID.findall(text)) - return ids - - -def resolve(href: str, page: str, site: str, base: str) -> str | None: - """Resolve an href to the built file it points at, or None if not ours.""" - target, _ = urldefrag(href) - target = unquote(target) - if not target: - return os.path.realpath(page) - if target.startswith("/"): - if not target.startswith(base): - return None - path = os.path.normpath(os.path.join(site, target[len(base):])) - else: - path = os.path.normpath(os.path.join(os.path.dirname(page), target)) - if not path.endswith(".html"): - path = os.path.join(path, "index.html") - return os.path.realpath(path) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("site", nargs="?", default="site") - parser.add_argument("--base", default="/halpi2/", - help="path component of site_url, for root-absolute links") - args = parser.parse_args() - - ids = collect_pages(args.site) - if not ids: - # Passing on an empty site would be a false green: the build produced - # nothing, or the path is wrong, and neither is "all anchors resolve". - print(f"No built pages found under {args.site!r} — nothing to check.", - file=sys.stderr) - return 2 - - broken: list[tuple[str, str, str]] = [] - checked = 0 - - for page in sorted(ids): - for href in HREF.findall(open(page, encoding="utf-8").read()): - if href.startswith(("http://", "https://", "mailto:", "data:")): - continue - _, fragment = urldefrag(href) - if not fragment: - continue - target = resolve(href, page, args.site, args.base) - if target is None: - continue - checked += 1 - relative = os.path.relpath(page, args.site) - if target not in ids: - broken.append((relative, href, "target page does not exist")) - elif unquote(fragment) not in ids[target]: - broken.append((relative, href, "no such anchor on the target page")) - - print(f"Checked {checked} anchor links across {len(ids)} pages.") - if broken: - print(f"\n{len(broken)} broken:\n") - for page, href, why in broken: - print(f" {page}\n -> {href} ({why})") - return 1 - print("All anchors resolve.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/check_glossary.py b/scripts/check_glossary.py deleted file mode 100644 index 2bf075d..0000000 --- a/scripts/check_glossary.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python3 -"""Check that a translation actually uses the terms its glossary prescribes. - -A glossary read before translating looks followed afterwards, because rereading -one's own text confirms whatever it already says. Every language branch so far -reached review with a term the glossary defines and the pages ignore — a second -name for the same connector, one page apart, which no reader can reconcile. - -The check is indirect but cheap: if a glossary term appears in the English -source and its prescribed translation appears nowhere in the target language, -some other word is doing that job. Run it before opening a pull request. - -It finds a term that is never used, not a term that has acquired a rival. German -says both `Spannungsausfall` and `Stromausfall` for *blackout* and passes here, -because the prescribed word does appear. Catching that needs the rival named, -which is what the glossary cannot know in advance. - -Exit status is 1 if any prescribed term is unused. -""" - -from __future__ import annotations - -import argparse -import re -import sys -import unicodedata -from pathlib import Path - -GLOSSARIES = { - "fi": "finnish-glossary.md", - "fr": "french-glossary.md", - "de": "german-glossary.md", - "sv": "swedish-glossary.md", - "es": "spanish-glossary.md", - "it": "italian-glossary.md", - "nl": "dutch-glossary.md", - "nb": "norwegian-glossary.md", - "da": "danish-glossary.md", -} - -ROW = re.compile(r"^\| *`?([^|`]+?)`? *\| *`?([^|`]+?)`? *\|") -SHORTEST_TERM = 5 -# Below this length a shared substring is more likely a compound than a clash. -BOUNDARY_FROM = 8 -# An English term used once may be phrased around; twice is a pattern. -MIN_ENGLISH_USES = 2 - - -def read_pages(directory: Path, only: set[str] | None = None) -> str: - """Concatenate a language's markdown with code and frontmatter removed. - - `only` restricts the read to a set of paths relative to the directory. The - English side is read that way so that a page nobody has translated cannot - make its vocabulary look missing: asking whether a translation uses a - prescribed term is meaningful only for pages that were translated at all. - """ - out = [] - for page in sorted(directory.rglob("*.md")): - if only is not None and page.relative_to(directory).as_posix() not in only: - continue - raw = page.read_text(encoding="utf-8") - text = re.sub(r"^---\n.*?\n---\n", "", raw, flags=re.S) - text = re.sub(r"```.*?```", " ", text, flags=re.S) - out.append(re.sub(r"`[^`\n]*`", " ", text)) - return fold("\n".join(out).lower()) - - -def terms(glossary: Path) -> list[tuple[str, str]]: - """Extract (english, translation) pairs from the glossary tables.""" - pairs = [] - for line in glossary.read_text(encoding="utf-8").splitlines(): - row = ROW.match(line) - if not row: - continue - english, translated = row.group(1).strip(), row.group(2).strip() - if english.lower().startswith("english") or set(english) <= set(":- "): - continue - pairs.append((english, translated)) - return pairs - - -def fold(text: str) -> str: - """Flatten the spelling differences that inflection introduces. - - Romance plurals move accents around — `tapón` becomes `tapones`, `imagen` - becomes `imágenes` — and Italian sets its apostrophe as U+2019 where a - glossary cell is typed with U+0027. Comparing the letters underneath keeps - those from reading as a term the pages never used. - """ - text = text.replace("’", "'").replace("ʼ", "'") - return "".join( - c for c in unicodedata.normalize("NFKD", text) if not unicodedata.combining(c) - ) - - -def alternatives(term: str) -> list[str]: - """Split a glossary cell into the forms that would each satisfy it.""" - term = re.sub(r"\s*\([^)]*\)", "", term).lower() - return [part.strip() for part in term.split("/") if part.strip()] - - -def inflectable(term: str) -> re.Pattern[str]: - """Match a term in whatever form a sentence needs. - - Every word may take an ending, not just the last one: Finnish inflects both - halves of `vapaa tila` and French pluralises both halves of `bouchon - obturateur`, so anchoring on the phrase as written finds neither. A verb - phrase also takes its object in the middle — `aseta CM5 uudelleen - paikalleen` — so a couple of words are allowed to intervene. - - The match must start at a word boundary, or a compounding language reports - a term as used when only a longer word containing it is present: Finnish - `virtalähde` (power supply) is a substring of `vakiovirtalähde` (constant - current source), two different components. Without the boundary this check - returns a false green, which is worse than a false alarm — a checker that - passes when it should not is no checker at all. - - The boundary only applies when the term starts with a word character. A row - like `−32 V and +32 V` opens with a minus sign, and `\\b` before a non-word - character asserts the opposite of what is meant — it would demand a letter - immediately before the minus and match nothing. - - It also only applies to terms long enough that a shared substring is - unlikely to be coincidence. A compounding language legitimately puts a short - common noun at the end of a compound — Danish `pakke` inside `salgspakke` - and `softwarepakke` is the same concept — so demanding a boundary there - reports correct prose as a violation. `virtalähde` at ten characters - appearing only inside `vakiovirtalähde` is a signal; `pakke` at five is not. - """ - words = [re.escape(w[: max(3, len(w) - 3)]) + r"\w*" for w in fold(term).split()] - body = r"(?:\W+\w+){0,2}\W+".join(words) - folded = fold(term) - long_enough = len(folded.replace(" ", "")) >= BOUNDARY_FROM - boundary = r"\b" if long_enough and re.match(r"\w", folded) else "" - return re.compile(boundary + body) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "language", choices=sorted(GLOSSARIES), help="target language code" - ) - parser.add_argument("--docs", default="docs", help="documentation root") - parser.add_argument( - "--glossaries", - default="solutions/translation", - help="directory holding the glossaries", - ) - args = parser.parse_args() - - target = Path(args.docs) / args.language - done = {p.relative_to(target).as_posix() for p in target.rglob("*.md")} - english = read_pages(Path(args.docs) / "en", only=done) - translated = read_pages(target) - glossary = Path(args.glossaries) / GLOSSARIES[args.language] - - checked, unused = 0, [] - for source, target in terms(glossary): - wanted, have = alternatives(source), alternatives(target) - # A row may offer several renderings — `plus (+) / miinus (−)`. Dropping - # only the ones too short to match reliably would leave the row - # demanding the survivors, so a page using `plusnapa` and never needing - # the negative half reads as a violation. If any alternative is too - # short to judge, the whole row is. - if not wanted or not have: - continue - if min(len(w) for w in wanted + have) < SHORTEST_TERM: - continue - uses = sum(english.count(w) for w in wanted) - if uses < MIN_ENGLISH_USES: - continue - checked += 1 - if not any(inflectable(h).search(translated) for h in have): - unused.append((source, target, uses)) - - print(f"Checked {checked} glossary terms against docs/{args.language}.") - if unused: - print(f"\n{len(unused)} prescribed but unused — something else took over:\n") - for source, target, uses in unused: - print(f" {source} -> {target} (English {uses}×, translation never)") - return 1 - print("Every prescribed term is in use.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/check_typography.py b/scripts/check_typography.py deleted file mode 100644 index 15c0985..0000000 --- a/scripts/check_typography.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -"""Count the typography rules a translation has to obey, per language. - -Written after two naive greps produced only false positives: searching for the -character pair »…« in Norwegian matches the gap *between* two correct «…» pairs, -and searching for a space before a colon matches English comments inside code -fences. Both looked like defects and neither was one. - -So quotations are checked by walking the marks in order and requiring them to -alternate open, close, open, close — which is what "the pairs are the right way -round" actually means — and everything is measured with code fences, inline code -and admonition syntax removed first. -""" - -from __future__ import annotations - -import re -import sys -from pathlib import Path - -# Which mark opens a quotation, and which closes it, per language. -QUOTES = { - "fi": ("”", "”"), # ”…” — same character both sides - "fr": ("«", "»"), # «…» - "de": ("„", "“"), # „…“ - "sv": ("”", "”"), # ”…” - "es": ("«", "»"), # «…» - "it": ("“", "”"), # “…” - "nl": ("“", "”"), # “…” - "nb": ("«", "»"), # «…» - "da": ("»", "«"), # »…« — outward, the opposite of Norwegian -} -# French is the one language that *requires* a space before ; : ! ? — and -# requires it to be unbreakable, so the line never breaks before the mark. -# Everywhere else any space there is an error, which is why this cannot be one -# rule for all: applying the French habit elsewhere is a known leak, and -# applying the majority rule to French would flag every correct sentence. -SPACE_REQUIRED = {"fr"} -PLAIN_SPACE_BEFORE_PUNCT = re.compile(r"\u0020[;:!?]") -# German compounds a multi-word proper name with hyphens throughout — -# NMEA-2000-Netzwerk, Signal-K-Server — and its glossary calls a missing hyphen -# there the most visible marker of a translation done by someone who does not -# write German. Every other language treats that same chain as an error, and a -# hyphen at the *junction* between a product name and a common noun -# (HaLOS-avbilder) is right in the Germanic languages and wrong in the Romance -# ones. One rule cannot serve all three cases, so each is scoped to where its -# glossary asks for it. -HYPHEN_CHAINS = re.compile(r"NMEA-2000|Signal-K|Raspberry-Pi|Compute-Module") -CHAINS_ALLOWED = {"de"} -JUNCTION_HYPHEN = re.compile( - r"\b(?:HALPI2|HaLOS|NMEA 2000|Signal K|Raspberry Pi|E7T)-" - r"[a-z\u00e1\u00e9\u00ed\u00f3\u00fa\u00f1\u00e0\u00e8\u00ec\u00f2\u00f9]" -) -JUNCTION_FORBIDDEN = {"es", "it"} -SPACE_BEFORE_PUNCT = re.compile(r"[   ][;:!?]") - - -def prose(text: str) -> str: - """The text a reader sees, with everything that is markup taken out. - - Inline code becomes a placeholder rather than nothing: deleting it joins the - words on either side and manufactures a space before the next punctuation - mark, which is exactly the false positive this function exists to avoid. - """ - text = re.sub(r"^---\n.*?\n---\n", "", text, flags=re.S) - text = re.sub(r"```.*?```", "\n", text, flags=re.S) - text = re.sub(r"`[^`\n]*`", "X", text) - text = re.sub(r'^!!! \w+ ".*"$', "", text, flags=re.M) # admonition syntax quotes - text = re.sub(r"\]\([^)]*\)", "]", text) # link targets - # A table's delimiter row carries the column alignment as colons — | ---: | - # — which reads as a space before a colon and is not prose at all. - text = re.sub(r"^[|\s:-]+$", "", text, flags=re.M) - # Repository names and filenames are identifiers that happen to contain - # hyphens — HALPI2-hardware, HALPI2-schematic_v0.6.1.pdf — and reading them - # as compounds of the target language invents defects that are not there. - text = re.sub(r"https?://\S+", "X", text) - text = re.sub( - r"\b[\w.-]+\.(?:pdf|zip|png|jpe?g|md|txt|json|ya?ml|step|bin|conf|sock)\b", - "X", - text, - ) - return text - - -def quotation_faults(text: str, opening: str, closing: str) -> list[str]: - """Marks must alternate open, close, open, close — and end closed.""" - if opening == closing: - count = text.count(opening) - return [] if count % 2 == 0 else [f"odd number of {opening} ({count})"] - faults, depth = [], 0 - for index, char in enumerate(text): - if char == opening: - if depth: - faults.append( - f"{opening} opens while already open: " - f"...{text[max(0, index - 40) : index + 20]}..." - ) - depth += 1 - elif char == closing: - if not depth: - faults.append( - f"{closing} closes nothing: " - f"...{text[max(0, index - 40) : index + 20]}..." - ) - else: - depth -= 1 - if depth: - faults.append(f"{depth} quotation(s) never closed") - return faults - - -def main() -> int: - languages = sys.argv[1:] or sorted(QUOTES) - worst = 0 - for language in languages: - opening, closing = QUOTES[language] - pages = sorted(Path("docs", language).rglob("*.md")) - quotes = spacing = chains = 0 - problems: list[str] = [] - for page in pages: - text = prose(page.read_text(encoding="utf-8")) - for fault in quotation_faults(text, opening, closing): - quotes += 1 - problems.append(f" {page}: {fault}") - rule = ( - PLAIN_SPACE_BEFORE_PUNCT - if language in SPACE_REQUIRED - else SPACE_BEFORE_PUNCT - ) - for match in rule.finditer(text): - spacing += 1 - wrong = "breakable space" if language in SPACE_REQUIRED else "space" - problems.append( - f" {page}: {wrong} before '{match.group()[-1]}': " - f"...{text[max(0, match.start() - 40):match.end() + 10]}..." - ) - allowed = language in CHAINS_ALLOWED - chain_rule = () if allowed else HYPHEN_CHAINS.finditer(text) - for match in chain_rule: - chains += 1 - problems.append( - f" {page}: hyphen inside a product name '{match.group()}'" - ) - if language in JUNCTION_FORBIDDEN: - for match in JUNCTION_HYPHEN.finditer(text): - chains += 1 - problems.append( - f" {page}: junction hyphen '{match.group()}' " - f"— not used in this language" - ) - - marks = sum(prose(p.read_text(encoding="utf-8")).count(opening) for p in pages) - status = "ok" if not problems else f"{len(problems)} PROBLEMS" - print( - f"{language}: {len(pages)} pages, {marks} quotations " - f"({opening}…{closing}), quote faults {quotes}, spacing {spacing}, " - f"hyphen chains {chains} — {status}" - ) - for problem in problems[:8]: - print(problem) - worst = max(worst, len(problems)) - return 1 if worst else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/map_anchors.py b/scripts/map_anchors.py deleted file mode 100644 index 7166e34..0000000 --- a/scripts/map_anchors.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 -"""Rewrite English anchor fragments in a translation to the translated slugs. - -Anchor slugs come from heading text, so a translated heading gets a different -slug and every link pointing at it breaks — including links on pages nobody -touched. Translators leave the English fragment in place; this maps it across. - -The mapping is positional: the structure comparison already proves the -translation has the same headings in the same order, so the nth heading of the -English page and the nth heading of the translation are the same heading. That -is stronger than matching on text, which cannot work once the text is in another -language. - -Usage: map_anchors.py [--apply] -Without --apply it only reports what it would change. -""" - -from __future__ import annotations - -import re -import sys -from pathlib import Path - -HEADING_ID = re.compile(r" list[str]: - """Heading ids of a built page, in document order. - - The default language has no URL segment of its own — `docs/en/index.md` is - served at the site root, not under `en/` — so English pages are looked up - without a prefix. - """ - stem = page[: -len(".md")] - stem = "" if stem == "index" else stem.removesuffix("/index") - prefix = "" if language == "en" else language - parts = [p for p in (prefix, stem) if p] - html = site.joinpath(*parts, "index.html") - if not html.exists(): - raise SystemExit( - f"No built page for {language}/{page} at {html} — build the site first." - ) - return HEADING_ID.findall(html.read_text(encoding="utf-8")) - - -def target_page(link: str, page: str) -> str | None: - """The markdown page a link points at, relative to the docs root.""" - path, _, _ = link.partition("#") - if link.startswith(("http://", "https://", "mailto:")): - return None - if not path: - return page - resolved = (Path(page).parent / path).as_posix() - resolved = Path(resolved).resolve().relative_to(Path.cwd().resolve()).as_posix() - return resolved if resolved.endswith(".md") else None - - -def main() -> int: - site, language = Path(sys.argv[1]), sys.argv[2] - apply = "--apply" in sys.argv - docs = Path("docs") - - english = { - p.relative_to(docs / "en").as_posix(): built_ids( - site, "en", p.relative_to(docs / "en").as_posix() - ) - for p in (docs / "en").rglob("*.md") - } - translated = {page: built_ids(site, language, page) for page in english} - - changes, unmapped = [], [] - for page in sorted(english): - source = docs / language / page - if not source.exists(): - continue - text = original = source.read_text(encoding="utf-8") - for link in set(LINK.findall(text)): - path, _, fragment = link.partition("#") - target = target_page(link, page) - if target is None or target not in english: - continue - ids_en, ids_tr = english[target], translated[target] - if fragment not in ids_en: - continue - if len(ids_en) != len(ids_tr): - unmapped.append( - f"{language}/{page} -> {link}: {target} has " - f"{len(ids_en)} headings in English, {len(ids_tr)} translated" - ) - continue - replacement = ids_tr[ids_en.index(fragment)] - if replacement != fragment: - text = text.replace(f"]({link})", f"]({path}#{replacement})") - changes.append( - f" {language}/{page}\n {fragment} -> {replacement}" - ) - if text != original: - if apply: - source.write_text(text, encoding="utf-8") - - verb = "rewritten" if apply else "to rewrite" - print(f"{len(changes)} anchors {verb} in docs/{language}.") - for change in changes: - print(change) - if unmapped: - print(f"\n{len(unmapped)} could not be mapped — structure differs:") - for problem in unmapped: - print(f" {problem}") - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/stamp_translation.py b/scripts/stamp_translation.py deleted file mode 100644 index 61fc83c..0000000 --- a/scripts/stamp_translation.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 -"""Write the translated_from stamp into a translation's frontmatter. - -Stamp a translation only when it has actually been (re-)translated against the -current English source. A stamp updated without real translation work reports -green and makes the staleness invisible — that is the one gap the status check -cannot close. - - uv run python scripts/stamp_translation.py docs/fi/user-guide/hardware.md -""" - -from __future__ import annotations - -import argparse -import subprocess -import sys -from pathlib import Path - -from translation_status import configured_languages - -DOCS = Path("docs") -STAMP_KEY = "translated_from" - - -def english_source(translation: Path, default: str) -> Path: - """docs// -> docs//.""" - parts = translation.parts - if len(parts) < 3 or parts[0] != DOCS.name: - raise SystemExit(f"{translation}: not a path under docs//") - if parts[1] == default: - raise SystemExit( - f"{translation}: this is a source page, not a translation. " - f"Source pages carry no stamp — that is the point: an English edit " - f"needs no ceremony." - ) - return DOCS / default / Path(*parts[2:]) - - -def blob_hash(path: Path) -> str: - return subprocess.run( - ["git", "hash-object", str(path)], - capture_output=True, text=True, check=True, - ).stdout.strip() - - -def restamp(text: str, value: str) -> str: - """Set the stamp, replacing an existing one and preserving other keys.""" - line = f"{STAMP_KEY}: {value}" - if not text.startswith("---\n"): - return f"---\n{line}\n---\n\n{text}" - end = text.find("\n---", 4) - if end == -1: - raise SystemExit("frontmatter is not terminated") - front, body = text[4:end], text[end + 4:].lstrip("\n") - kept = [l for l in front.splitlines() if not l.startswith(f"{STAMP_KEY}:")] - return "---\n" + "\n".join([*kept, line]) + "\n---\n\n" + body - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("translations", nargs="+", type=Path) - args = parser.parse_args() - - default, _ = configured_languages() - for translation in args.translations: - if not translation.exists(): - raise SystemExit(f"{translation}: does not exist") - source = english_source(translation, default) - if not source.exists(): - raise SystemExit(f"{translation}: no English source at {source}") - value = blob_hash(source) - translation.write_text( - restamp(translation.read_text(encoding="utf-8"), value), - encoding="utf-8", - ) - print(f"{translation}: {STAMP_KEY} = {value}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/translation_status.py b/scripts/translation_status.py deleted file mode 100644 index ff22ea3..0000000 --- a/scripts/translation_status.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -"""Report which translations are missing or out of date. - -A translation records the git blob hash of the English source it was written -against, in its own frontmatter: - - --- - translated_from: at translation time> - --- - -The English page carries nothing, so an English edit needs no ceremony: editing -it changes its content, which changes its hash, which makes every translation of -it report as stale on its own. - -Reports; never blocks. Exit status is 0 unless the check itself could not run. -""" - -from __future__ import annotations - -import argparse -import subprocess -import sys -import tempfile -from dataclasses import dataclass -from pathlib import Path - -import yaml - -DOCS = Path("docs") -STAMP_KEY = "translated_from" - - -class _Loader(yaml.SafeLoader): - """mkdocs.yml carries python/name tags that SafeLoader refuses to parse.""" - - -_Loader.add_multi_constructor("", lambda loader, suffix, node: None) - - -def configured_languages() -> tuple[str, list[str]]: - """Return (default language, other languages) from the i18n plugin config.""" - config = yaml.load(Path("mkdocs.yml").read_text(encoding="utf-8"), Loader=_Loader) - for plugin in config.get("plugins", []): - if isinstance(plugin, dict) and "i18n" in plugin: - languages = plugin["i18n"]["languages"] - default = next(l["locale"] for l in languages if l.get("default")) - others = [l["locale"] for l in languages if not l.get("default")] - return default, others - raise SystemExit("mkdocs.yml has no i18n plugin configuration") - - -def blob_hash(path: Path) -> str: - return subprocess.run( - ["git", "hash-object", str(path)], - capture_output=True, text=True, check=True, - ).stdout.strip() - - -def stamp_of(path: Path) -> str | None: - """Read translated_from from a page's frontmatter, if it has one.""" - text = path.read_text(encoding="utf-8") - if not text.startswith("---\n"): - return None - end = text.find("\n---", 4) - if end == -1: - return None - front = yaml.safe_load(text[4:end]) or {} - value = front.get(STAMP_KEY) - return str(value) if value else None - - -def english_diff(stamped: str, current: Path) -> str | None: - """Diff the stamped English blob against the English page as it stands now. - - The current page is compared from the working tree rather than as a stored - object: `git hash-object` computes a hash without writing the object, so - diffing two hashes would fail on the side that was never stored. - """ - blob = subprocess.run( - ["git", "cat-file", "-p", stamped], capture_output=True, text=True, - ) - if blob.returncode != 0: - return None # stamped blob not in this clone — CI needs fetch-depth: 0 - with tempfile.TemporaryDirectory() as tmp: - was = Path(tmp) / current.name - was.write_text(blob.stdout, encoding="utf-8") - result = subprocess.run( - ["git", "diff", "--no-index", "--no-color", str(was), str(current)], - capture_output=True, text=True, - ) - # --no-index exits 1 when the files differ, which is the expected case. - # Drop the file headers: they carry a temporary path, and the page is - # already named in the surrounding report. - noise = ("diff --git ", "index ", "--- ", "+++ ") - return "\n".join( - line for line in result.stdout.splitlines() - if not line.startswith(noise) - ) - - -@dataclass -class Entry: - language: str - page: str # path relative to the language directory - state: str # missing | unstamped | stale | orphaned | current - expected: str # blob hash the translation should record - diff: str | None = None - - -def collect(default: str, languages: list[str], want_diff: bool) -> list[Entry]: - sources = sorted(p for p in (DOCS / default).rglob("*.md")) - entries: list[Entry] = [] - for source in sources: - relative = source.relative_to(DOCS / default) - expected = blob_hash(source) - for language in languages: - target = DOCS / language / relative - if not target.exists(): - entries.append(Entry(language, str(relative), "missing", expected)) - continue - stamped = stamp_of(target) - if stamped is None: - entries.append(Entry(language, str(relative), "unstamped", expected)) - elif stamped == expected: - entries.append(Entry(language, str(relative), "current", expected)) - else: - diff = english_diff(stamped, source) if want_diff else None - entries.append(Entry(language, str(relative), "stale", expected, diff)) - - # A translation whose source was deleted is invisible to the loop above, - # because that walks the sources. It is still a page being served. - for language in languages: - root = DOCS / language - for translation in sorted(root.rglob("*.md")): - if not (DOCS / default / translation.relative_to(root)).exists(): - entries.append( - Entry(language, str(translation.relative_to(root)), "orphaned", "") - ) - return entries - - -def render_text(entries: list[Entry]) -> str: - out = [] - for language in sorted({e.language for e in entries}): - rows = [e for e in entries if e.language == language] - counts = {s: sum(1 for e in rows if e.state == s) for s in - ("current", "stale", "unstamped", "missing", "orphaned")} - out.append(f"{language}: " + " ".join(f"{k}={v}" for k, v in counts.items())) - for entry in rows: - if entry.state != "current": - out.append(f" {entry.state:9s} {entry.page}") - if entry.expected: - out.append(f" {STAMP_KEY}: {entry.expected}") - return "\n".join(out) - - -def render_markdown(entries: list[Entry], only: set[str] | None) -> str: - shown = [e for e in entries if only is None or e.page in only] - out = ["## Translation status", ""] - for language in sorted({e.language for e in entries}): - rows = [e for e in entries if e.language == language] - counts = {s: sum(1 for e in rows if e.state == s) for s in - ("current", "stale", "unstamped", "missing", "orphaned")} - summary = ", ".join(f"{v} {k}" for k, v in counts.items() if v) - out.append(f"**{language}** — {summary}") - out.append("") - - behind = [e for e in shown if e.state != "current"] - if not behind: - out.append("Every translation of the pages in scope is current.") - return "\n".join(out) - - out += ["| Language | Page | State | Stamp to record |", - "|:---|:---|:---|:---|"] - for entry in behind: - out.append(f"| {entry.language} | `{entry.page}` | {entry.state} | `{entry.expected}` |") - out.append("") - - for entry in behind: - if entry.diff: - out += [f"
English changes since " - f"{entry.language}/{entry.page} was translated", - "", "```diff", entry.diff.rstrip(), "```", "", "
", ""] - elif entry.state == "stale": - out.append(f"") - return "\n".join(out) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--format", choices=("text", "markdown"), default="text") - parser.add_argument("--diff", action="store_true", - help="include the English diff for stale pages") - parser.add_argument("--only-pages", nargs="*", metavar="PATH", - help="restrict the detail section to these docs//-relative paths") - args = parser.parse_args() - - default, languages = configured_languages() - if not languages: - print("No translation languages configured.") - return 0 - - entries = collect(default, languages, want_diff=args.diff) - if args.format == "markdown": - only = set(args.only_pages) if args.only_pages else None - print(render_markdown(entries, only)) - else: - print(render_text(entries)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/solutions/translation/danish-glossary.md b/solutions/translation/danish-glossary.md index 83a65ae..5164363 100644 --- a/solutions/translation/danish-glossary.md +++ b/solutions/translation/danish-glossary.md @@ -572,10 +572,10 @@ prescribes for a proper name: `HAT-stikliste`, `CAN HAT-stiklisten`, A translated page is not done until: 1. `uv run mkdocs build --strict` passes. -2. `uv run python scripts/check_anchors.py site` passes. -3. `uv run python scripts/translation_status.py` shows the page as current. -4. `uv run python scripts/check_glossary.py da` passes. -5. `uv run python scripts/check_typography.py da` passes. +2. `uv run check-anchors site` passes. +3. `uv run translation-status` shows the page as current. +4. `uv run check-glossary da` passes. +5. `uv run check-typography da` passes. 6. Structure matches the source — see `.claude/skills/translate-page/SKILL.md`. 7. Every number in the English text appears in the translation. A wrong voltage or current in an installation guide is a safety problem, not a typo. diff --git a/solutions/translation/dutch-glossary.md b/solutions/translation/dutch-glossary.md index 4e3e3d7..75f1c52 100644 --- a/solutions/translation/dutch-glossary.md +++ b/solutions/translation/dutch-glossary.md @@ -595,13 +595,13 @@ the English does. `we` is not the informal address rule 1 forbids; `je` and A translated page is not done until: 1. `uv run mkdocs build --strict` passes. -2. `uv run python scripts/check_anchors.py site --base /sh-rpi/` passes. The +2. `uv run check-anchors site --base /sh-rpi/` passes. The script's default `--base` is `/halpi2/`, which is wrong in this repository — `site_url` here ends in `/sh-rpi`, so pass it explicitly or every root-absolute link is reported as broken. -3. `uv run python scripts/translation_status.py` shows the page as current. -4. `uv run python scripts/check_glossary.py nl` passes. -5. `uv run python scripts/check_typography.py nl` passes. +3. `uv run translation-status` shows the page as current. +4. `uv run check-glossary nl` passes. +5. `uv run check-typography nl` passes. 6. Structure matches the source — see `.claude/skills/translate-page/SKILL.md`. 7. **The seven rules at the top are counted against the pages, not re-read.** diff --git a/solutions/translation/french-glossary.md b/solutions/translation/french-glossary.md index a97ff32..6351780 100644 --- a/solutions/translation/french-glossary.md +++ b/solutions/translation/french-glossary.md @@ -462,12 +462,12 @@ updated in the same change rather than looking like a translation error. A translated page is not done until: 1. `uv run mkdocs build --strict` passes — the same command CI runs. -2. `uv run python scripts/check_typography.py` passes for `fr`. French is the +2. `uv run check-typography` passes for `fr`. French is the one language that *requires* the space before `; : ! ?`, and the checker knows it — a plain U+0020 there is reported. -3. `uv run python scripts/check_glossary.py fr` passes: every term this glossary +3. `uv run check-glossary fr` passes: every term this glossary prescribes and the English pages actually use appears in the French. -4. `uv run python scripts/check_anchors.py site` passes. +4. `uv run check-anchors site` passes. 5. `uv run mkdocs serve` shows the page rendering correctly, with lists as lists — always leave a blank line before and after a list. 6. Every term used on the page that appears in this glossary matches it. @@ -475,6 +475,6 @@ A translated page is not done until: ## Related - `finnish-glossary.md` — the sibling glossary and the general approach -- `../../scripts/check_glossary.py`, `../../scripts/check_typography.py` — the - two checks that read this file +- `check-glossary`, `check-typography` — the two checks that read this file, + from the `halos-docs-tools` package - mkdocs-static-i18n documentation: https://ultrabug.github.io/mkdocs-static-i18n/ diff --git a/solutions/translation/italian-glossary.md b/solutions/translation/italian-glossary.md index 30378a7..0dd6d34 100644 --- a/solutions/translation/italian-glossary.md +++ b/solutions/translation/italian-glossary.md @@ -569,16 +569,14 @@ that inherits a known error. This glossary row is the record. A translated page is not done until: 1. `uv run mkdocs build --strict` passes — the same command CI runs. -2. `uv run python scripts/check_anchors.py site` passes. -3. `uv run python scripts/check_typography.py it` passes. The script already +2. `uv run check-anchors site` passes. +3. `uv run check-typography it` passes. The script already knows Italian: `QUOTES["it"]` is `("“", "”")`, Italian is **not** in `SPACE_REQUIRED`, and it is not in `CHAINS_ALLOWED`, so rules 2, 3 and 4 are machine-checked. -4. `uv run python scripts/check_glossary.py it` passes. `it` is already - registered in the `GLOSSARIES` dict as `italian-glossary.md`, so no change to - the script is needed — unlike the HALPI2 repository, where registering the - language was a prerequisite for the Italian branch. -5. `uv run python scripts/translation_status.py` shows the page as current. +4. `uv run check-glossary it` passes. The checker carries `it` and maps it to + `italian-glossary.md`, so this needs no setup. +5. `uv run translation-status` shows the page as current. 6. `uv run mkdocs serve` shows the page rendering correctly in the browser, with lists as lists (see `../best-practices/markdown-lists-need-blank-line-2026-05-16.md` — the diff --git a/solutions/translation/norwegian-glossary.md b/solutions/translation/norwegian-glossary.md index 8365083..94006db 100644 --- a/solutions/translation/norwegian-glossary.md +++ b/solutions/translation/norwegian-glossary.md @@ -67,7 +67,7 @@ Read this section before anything else. Every one of these is stated the opposite way in at least one sibling glossary. Danish is the dangerous neighbour: it is close enough to read as correct and is being written against the same English source, so a Danish habit that slips in will not look wrong to -anyone who is not counting. `scripts/check_glossary.py` already registers `da` +anyone who is not counting. `check-glossary` already registers `da` alongside `nb`, and machine translation produces Danish-shaped Norwegian on its own, so the rule holds whether or not a Danish page exists yet. @@ -546,14 +546,14 @@ in the stack; render that as `kortet under` rather than as either term above. A translated page is not done until: 1. `uv run mkdocs build --strict` passes — the same command CI runs. -2. `uv run python scripts/check_anchors.py site` passes. -3. `uv run python scripts/translation_status.py` shows the page as current. -4. `uv run python scripts/check_glossary.py nb` passes. -5. `uv run python scripts/check_typography.py nb` passes. +2. `uv run check-anchors site` passes. +3. `uv run translation-status` shows the page as current. +4. `uv run check-glossary nb` passes. +5. `uv run check-typography nb` passes. 6. Structure matches the source — see `.claude/skills/translate-page/SKILL.md`. 7. Every term used on the page that appears in this glossary matches it. -Both `check_glossary.py` and `check_typography.py` already register `nb` +Both `check-glossary` and `check-typography` already register `nb` (`"nb": "norwegian-glossary.md"` and `"nb": ("«", "»")`), so no script change is needed before the first page — unlike in the HALPI2 repository, where the first translator had to add that line. @@ -564,7 +564,7 @@ translator had to add that line. whatever it already says. Both the French and German branches shipped a half-applied typography rule to review for exactly this reason, and Danish is close enough to Norwegian that a leak from a parallel branch will read as fine. -`check_typography.py nb` does the counting properly, with code fences stripped; +`check-typography nb` does the counting properly, with code fences stripped; the greps below are for spot checks while writing. Act on any non-zero count. | Rule | Command | Expected | @@ -606,7 +606,7 @@ typo. The last row is not optional. Inherited from the HALPI2 pages, where the page translators reported them and they were consolidated here rather than written by each of them. They are kept because the vocabulary is shared; a row whose English term does not occur in the -SH-RPi pages simply goes unchecked by `check_glossary.py`. **New SH-RPi terms +SH-RPi pages simply goes unchecked by `check-glossary`. **New SH-RPi terms belong under `## SH-RPi terms`**, not here, so the two products' additions stay distinguishable. diff --git a/solutions/translation/spanish-glossary.md b/solutions/translation/spanish-glossary.md index ef3fe0e..8098e19 100644 --- a/solutions/translation/spanish-glossary.md +++ b/solutions/translation/spanish-glossary.md @@ -574,10 +574,10 @@ pages is exactly the drift this file exists to prevent. A translated page is not done until: 1. `uv run mkdocs build --strict` passes — the same command CI runs. -2. `uv run python scripts/check_anchors.py site` passes. -3. `uv run python scripts/translation_status.py` shows the page as current. -4. `uv run python scripts/check_glossary.py es` passes. -5. `uv run python scripts/check_typography.py es` passes — it walks the `«…»` +2. `uv run check-anchors site` passes. +3. `uv run translation-status` shows the page as current. +4. `uv run check-glossary es` passes. +5. `uv run check-typography es` passes — it walks the `«…»` marks in order and measures the space-before-punctuation rule with code fences removed, which a naive grep cannot do. 6. Structure matches the source — see `.claude/skills/translate-page/SKILL.md`. diff --git a/solutions/translation/swedish-glossary.md b/solutions/translation/swedish-glossary.md index 6fb0a29..77291f0 100644 --- a/solutions/translation/swedish-glossary.md +++ b/solutions/translation/swedish-glossary.md @@ -441,11 +441,11 @@ numbers. Check which board the page is about before reaching for either word. A translated page is not done until: 1. `uv run mkdocs build --strict` passes — the same command CI runs. -2. `uv run python scripts/check_anchors.py site` passes. -3. `uv run python scripts/check_glossary.py sv` passes — it reads the tables in +2. `uv run check-anchors site` passes. +3. `uv run check-glossary sv` passes — it reads the tables in this file, so a row added here is checked from then on. -4. `uv run python scripts/check_typography.py sv` passes. -5. `uv run python scripts/translation_status.py` shows the page as current. +4. `uv run check-typography sv` passes. +5. `uv run translation-status` shows the page as current. 6. Every term used on the page that appears in this glossary matches it. 7. **The four rules at the top are tested against the pages, not re-read.** A half-applied typography rule looks followed when you read it. Both the French diff --git a/uv.lock b/uv.lock index a0feaf6..499d671 100644 --- a/uv.lock +++ b/uv.lock @@ -156,6 +156,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, ] +[[package]] +name = "halos-docs-tools" +version = "0.1.0" +source = { git = "https://github.com/halos-org/docs-tools?rev=v0.1.0#7f09d05f54cf64184a7a4d7205c166bc49ec0f6e" } +dependencies = [ + { name = "pyyaml" }, +] + [[package]] name = "idna" version = "3.11" @@ -508,6 +516,7 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "click" }, + { name = "halos-docs-tools" }, { name = "mkdocs-material" }, { name = "mkdocs-static-i18n" }, ] @@ -515,6 +524,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "click", specifier = "<8.3" }, + { name = "halos-docs-tools", git = "https://github.com/halos-org/docs-tools?rev=v0.1.0" }, { name = "mkdocs-material", specifier = ">=9.5" }, { name = "mkdocs-static-i18n", specifier = ">=1.2" }, ]