From dc63dfef9e006294120b21a20078c15d2e824c4b Mon Sep 17 00:00:00 2001 From: Matti Airas Date: Thu, 13 Aug 2026 17:47:48 +0300 Subject: [PATCH 1/2] feat(ci): adopt halos-docs-tools and the shared gate The six checkers move out of scripts/ and into the halos-docs-tools package, pinned at v0.1.0. The workflow shrinks to the caller stanza for halos-org/shared-workflows, which runs the same commands and, unlike the advisory version it replaces, fails the run when a translation is stale, missing, unstamped or orphaned. One commit rather than a chain: the old workflow invokes the scripts by path, so deleting them and repointing CI cannot be separated without an intermediate commit whose CI is broken. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/skills/translate-page/SKILL.md | 18 +- .github/workflows/translation-status.yml | 91 ++-------- pyproject.toml | 1 + scripts/check_anchors.py | 100 ----------- scripts/check_glossary.py | 179 ------------------- scripts/check_typography.py | 158 ----------------- scripts/map_anchors.py | 115 ------------ scripts/stamp_translation.py | 81 --------- scripts/translation_status.py | 214 ----------------------- uv.lock | 10 ++ 10 files changed, 32 insertions(+), 935 deletions(-) delete mode 100644 scripts/check_anchors.py delete mode 100644 scripts/check_glossary.py delete mode 100644 scripts/check_typography.py delete mode 100644 scripts/map_anchors.py delete mode 100644 scripts/stamp_translation.py delete mode 100644 scripts/translation_status.py diff --git a/.claude/skills/translate-page/SKILL.md b/.claude/skills/translate-page/SKILL.md index 763c5c3..e6ffec6 100644 --- a/.claude/skills/translate-page/SKILL.md +++ b/.claude/skills/translate-page/SKILL.md @@ -11,7 +11,7 @@ exist because each of them was broken once and cost real work. ## Inputs - A page path under `docs/en/`, or a page reported by - `uv run python scripts/translation_status.py` as `missing` or `stale`. + `uv run translation-status` as `missing` or `stale`. - A target language directory, e.g. `docs/fi/`. ## Before translating @@ -79,7 +79,7 @@ The stamp records the git blob hash of the English source the translation was written against. Write it with the helper, never by hand: ```bash -uv run python scripts/stamp_translation.py docs/fi/hardware/index.md +uv run stamp-translation docs/fi/hardware/index.md ``` **Stamp only when you have actually translated.** A stamp updated without real @@ -121,18 +121,18 @@ All four, every time: ```bash uv run mkdocs build --strict -uv run python scripts/check_anchors.py site -uv run python scripts/translation_status.py -uv run python scripts/check_glossary.py fi -uv run python scripts/check_typography.py fi +uv run check-anchors site +uv run translation-status +uv run check-glossary fi +uv run check-typography fi ``` **Leave every anchor fragment in its English form while translating**, then map them all at once once the language is complete and the site has been built: ```bash -uv run python scripts/map_anchors.py site fi # report -uv run python scripts/map_anchors.py site fi --apply # rewrite +uv run map-anchors site fi # report +uv run map-anchors site fi --apply # rewrite ``` The mapping is positional — the nth heading of the English page and the nth @@ -144,7 +144,7 @@ the text is in another language. whatever they already say, so the terminology looks consistent right up until a reviewer finds the same connector under two names on adjacent pages. Every language so far shipped that mistake, and each time it landed on the last pages -translated, once the glossary had stopped being opened. `check_glossary.py` +translated, once the glossary had stopped being opened. `check-glossary` reports terms the glossary prescribes and the pages never use — the signature of a rival word having quietly taken over. diff --git a/.github/workflows/translation-status.yml b/.github/workflows/translation-status.yml index e21dd3b..7ddedbf 100644 --- a/.github/workflows/translation-status.yml +++ b/.github/workflows/translation-status.yml @@ -1,95 +1,28 @@ name: Translation Status +# No paths filter. The gate is a property of the whole repository, not of a +# diff, and a required check that never runs on a pull request touching none of +# the filtered paths leaves that pull request unmergeable forever. on: pull_request: - paths: - - 'docs/**' - - 'mkdocs.yml' - - 'scripts/**' push: branches: [main] workflow_dispatch: +# The called workflow inherits this token, so the comment needs +# pull-requests: write here. Omit it and the run still gates; only the comment +# is skipped. permissions: contents: read pull-requests: write concurrency: group: translation-status-${{ github.ref }} - cancel-in-progress: true + # Pull requests only. On push, github.ref is refs/heads/main for every merge, + # so cancelling lets one merge kill the run checking the one before it -- and + # a cancelled run is grey, not red, so nobody is told. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - status: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # Full history: the report resolves the stamped blob to show the - # English diff since a translation was written. - fetch-depth: 0 - - - uses: astral-sh/setup-uv@v5 - - run: uv sync - - - name: Report translation status - run: | - # tee, not plain redirection: a report only in the job summary is - # invisible in the logs, which is where you look when it misbehaves. - uv run python scripts/translation_status.py --format markdown --diff \ - | tee report.md - cat report.md >> "$GITHUB_STEP_SUMMARY" - - - name: Comment on the pull request - if: github.event_name == 'pull_request' - env: - GH_TOKEN: ${{ github.token }} - PR: ${{ github.event.number }} - run: | - # Only the English pages this PR actually touches. Which paths a PR - # touched is a fact, so a PR editing only translations says nothing. - pages=$(git diff --name-only \ - "origin/${{ github.base_ref }}...HEAD" -- 'docs/en/**/*.md' \ - | sed 's|^docs/en/||') - if [ -z "$pages" ]; then - echo "No English pages touched; nothing to report." - exit 0 - fi - - # shellcheck disable=SC2086 - uv run python scripts/translation_status.py \ - --format markdown --diff --only-pages $pages > comment.md - printf '\n\n' >> comment.md - - existing=$(gh api "repos/${{ github.repository }}/issues/$PR/comments" \ - --jq 'map(select(.body | contains(""))) | .[0].id // empty') - if [ -n "$existing" ]; then - gh api "repos/${{ github.repository }}/issues/comments/$existing" \ - -X PATCH -F body=@comment.md --silent - echo "Updated comment $existing" - else - gh api "repos/${{ github.repository }}/issues/$PR/comments" \ - -F body=@comment.md --silent - echo "Created comment" - fi - - # Last, because unlike a stale translation a broken anchor is actual - # breakage and fails the run — and the report above must still be - # published when it does. - - name: Check anchors - run: | - uv run mkdocs build --strict - # PIPESTATUS, not $?: piping into tee would otherwise mask the - # checker's exit status behind tee's. - set +e - uv run python scripts/check_anchors.py site | tee anchors.txt - broken=${PIPESTATUS[0]} - set -e - { - echo "" - echo "## Anchor check" - echo "" - echo '```' - cat anchors.txt - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - exit "$broken" + translation-status: + uses: halos-org/shared-workflows/.github/workflows/translation-status.yml@main diff --git a/pyproject.toml b/pyproject.toml index afb851f..fd54317 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,4 +7,5 @@ dependencies = [ "mkdocs-material>=9.5", "click<8.3", "mkdocs-static-i18n>=1.2", + "halos-docs-tools @ git+https://github.com/halos-org/docs-tools@v0.1.0", ] diff --git a/scripts/check_anchors.py b/scripts/check_anchors.py deleted file mode 100644 index 8ad9591..0000000 --- a/scripts/check_anchors.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -"""Verify that every internal anchor in the built site resolves to a real id. - -Anchors are generated from heading text, so translating a heading changes its -slug and silently breaks every link pointing at it — including links on pages -that were not touched, which is why this is a delayed fault: a cross-page anchor -keeps working until its *target* page is translated. `mkdocs build --strict` -does not validate anchors at all. - -Run against a built site directory. Exit status is 1 if any anchor is broken. -""" - -from __future__ import annotations - -import argparse -import os -import re -import sys -from urllib.parse import unquote, urldefrag - -HREF = re.compile(r'href="([^"]+)"') -ID = re.compile(r'\sid="([^"]+)"') - - -def collect_pages(site: str) -> dict[str, set[str]]: - """Map each built page to the set of element ids it defines.""" - ids: dict[str, set[str]] = {} - for root, _, files in os.walk(site): - for name in files: - if name.endswith(".html"): - path = os.path.join(root, name) - text = open(path, encoding="utf-8").read() - ids[os.path.realpath(path)] = set(ID.findall(text)) - return ids - - -def resolve(href: str, page: str, site: str, base: str) -> str | None: - """Resolve an href to the built file it points at, or None if not ours.""" - target, _ = urldefrag(href) - target = unquote(target) - if not target: - return os.path.realpath(page) - if target.startswith("/"): - if not target.startswith(base): - return None - path = os.path.normpath(os.path.join(site, target[len(base):])) - else: - path = os.path.normpath(os.path.join(os.path.dirname(page), target)) - if not path.endswith(".html"): - path = os.path.join(path, "index.html") - return os.path.realpath(path) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("site", nargs="?", default="site") - parser.add_argument("--base", default="/halpi2/", - help="path component of site_url, for root-absolute links") - args = parser.parse_args() - - ids = collect_pages(args.site) - if not ids: - # Passing on an empty site would be a false green: the build produced - # nothing, or the path is wrong, and neither is "all anchors resolve". - print(f"No built pages found under {args.site!r} — nothing to check.", - file=sys.stderr) - return 2 - - broken: list[tuple[str, str, str]] = [] - checked = 0 - - for page in sorted(ids): - for href in HREF.findall(open(page, encoding="utf-8").read()): - if href.startswith(("http://", "https://", "mailto:", "data:")): - continue - _, fragment = urldefrag(href) - if not fragment: - continue - target = resolve(href, page, args.site, args.base) - if target is None: - continue - checked += 1 - relative = os.path.relpath(page, args.site) - if target not in ids: - broken.append((relative, href, "target page does not exist")) - elif unquote(fragment) not in ids[target]: - broken.append((relative, href, "no such anchor on the target page")) - - print(f"Checked {checked} anchor links across {len(ids)} pages.") - if broken: - print(f"\n{len(broken)} broken:\n") - for page, href, why in broken: - print(f" {page}\n -> {href} ({why})") - return 1 - print("All anchors resolve.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/check_glossary.py b/scripts/check_glossary.py deleted file mode 100644 index 8f5dbac..0000000 --- a/scripts/check_glossary.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python3 -"""Check that a translation actually uses the terms its glossary prescribes. - -A glossary read before translating looks followed afterwards, because rereading -one's own text confirms whatever it already says. Every language branch so far -reached review with a term the glossary defines and the pages ignore — a second -name for the same connector, one page apart, which no reader can reconcile. - -The check is indirect but cheap: if a glossary term appears in the English -source and its prescribed translation appears nowhere in the target language, -some other word is doing that job. Run it before opening a pull request. - -It finds a term that is never used, not a term that has acquired a rival. German -says both `Spannungsausfall` and `Stromausfall` for *blackout* and passes here, -because the prescribed word does appear. Catching that needs the rival named, -which is what the glossary cannot know in advance. - -Exit status is 1 if any prescribed term is unused. -""" - -from __future__ import annotations - -import argparse -import re -import sys -import unicodedata -from pathlib import Path - -GLOSSARIES = { - "fi": "finnish-glossary.md", -} - -ROW = re.compile(r"^\| *`?([^|`]+?)`? *\| *`?([^|`]+?)`? *\|") -SHORTEST_TERM = 5 -# Below this length a shared substring is more likely a compound than a clash. -BOUNDARY_FROM = 8 -# An English term used once may be phrased around; twice is a pattern. -MIN_ENGLISH_USES = 2 - - -def read_pages(directory: Path, only: set[str] | None = None) -> str: - """Concatenate a language's markdown with code and frontmatter removed. - - `only` restricts the read to a set of paths relative to the directory. The - English side is read that way so that a page nobody has translated cannot - make its vocabulary look missing: asking whether a translation uses a - prescribed term is meaningful only for pages that were translated at all. - """ - out = [] - for page in sorted(directory.rglob("*.md")): - if only is not None and page.relative_to(directory).as_posix() not in only: - continue - raw = page.read_text(encoding="utf-8") - text = re.sub(r"^---\n.*?\n---\n", "", raw, flags=re.S) - text = re.sub(r"```.*?```", " ", text, flags=re.S) - out.append(re.sub(r"`[^`\n]*`", " ", text)) - return fold("\n".join(out).lower()) - - -def terms(glossary: Path) -> list[tuple[str, str]]: - """Extract (english, translation) pairs from the glossary tables.""" - pairs = [] - for line in glossary.read_text(encoding="utf-8").splitlines(): - row = ROW.match(line) - if not row: - continue - english, translated = row.group(1).strip(), row.group(2).strip() - if english.lower().startswith("english") or set(english) <= set(":- "): - continue - pairs.append((english, translated)) - return pairs - - -def fold(text: str) -> str: - """Flatten the spelling differences that inflection introduces. - - Romance plurals move accents around — `tapón` becomes `tapones`, `imagen` - becomes `imágenes` — and Italian sets its apostrophe as U+2019 where a - glossary cell is typed with U+0027. Comparing the letters underneath keeps - those from reading as a term the pages never used. - """ - text = text.replace("’", "'").replace("ʼ", "'") - return "".join( - c for c in unicodedata.normalize("NFKD", text) if not unicodedata.combining(c) - ) - - -def alternatives(term: str) -> list[str]: - """Split a glossary cell into the forms that would each satisfy it.""" - term = re.sub(r"\s*\([^)]*\)", "", term).lower() - return [part.strip() for part in term.split("/") if part.strip()] - - -def inflectable(term: str) -> re.Pattern[str]: - """Match a term in whatever form a sentence needs. - - Every word may take an ending, not just the last one: Finnish inflects both - halves of `vapaa tila` and French pluralises both halves of `bouchon - obturateur`, so anchoring on the phrase as written finds neither. A verb - phrase also takes its object in the middle — `aseta CM5 uudelleen - paikalleen` — so a couple of words are allowed to intervene. - - The match must start at a word boundary, or a compounding language reports - a term as used when only a longer word containing it is present: Finnish - `virtalähde` (power supply) is a substring of `vakiovirtalähde` (constant - current source), two different components. Without the boundary this check - returns a false green, which is worse than a false alarm — a checker that - passes when it should not is no checker at all. - - The boundary only applies when the term starts with a word character. A row - like `−32 V and +32 V` opens with a minus sign, and `\\b` before a non-word - character asserts the opposite of what is meant — it would demand a letter - immediately before the minus and match nothing. - - It also only applies to terms long enough that a shared substring is - unlikely to be coincidence. A compounding language legitimately puts a short - common noun at the end of a compound — Danish `pakke` inside `salgspakke` - and `softwarepakke` is the same concept — so demanding a boundary there - reports correct prose as a violation. `virtalähde` at ten characters - appearing only inside `vakiovirtalähde` is a signal; `pakke` at five is not. - """ - words = [re.escape(w[: max(3, len(w) - 3)]) + r"\w*" for w in fold(term).split()] - body = r"(?:\W+\w+){0,2}\W+".join(words) - folded = fold(term) - long_enough = len(folded.replace(" ", "")) >= BOUNDARY_FROM - boundary = r"\b" if long_enough and re.match(r"\w", folded) else "" - return re.compile(boundary + body) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "language", choices=sorted(GLOSSARIES), help="target language code" - ) - parser.add_argument("--docs", default="docs", help="documentation root") - parser.add_argument( - "--glossaries", - default="solutions/translation", - help="directory holding the glossaries", - ) - args = parser.parse_args() - - target = Path(args.docs) / args.language - done = {p.relative_to(target).as_posix() for p in target.rglob("*.md")} - english = read_pages(Path(args.docs) / "en", only=done) - translated = read_pages(target) - glossary = Path(args.glossaries) / GLOSSARIES[args.language] - - checked, unused = 0, [] - for source, target in terms(glossary): - wanted, have = alternatives(source), alternatives(target) - # A row may offer several renderings — `plus (+) / miinus (−)`. Dropping - # only the ones too short to match reliably would leave the row - # demanding the survivors, so a page using `plusnapa` and never needing - # the negative half reads as a violation. If any alternative is too - # short to judge, the whole row is. - if not wanted or not have: - continue - if min(len(w) for w in wanted + have) < SHORTEST_TERM: - continue - uses = sum(english.count(w) for w in wanted) - if uses < MIN_ENGLISH_USES: - continue - checked += 1 - if not any(inflectable(h).search(translated) for h in have): - unused.append((source, target, uses)) - - print(f"Checked {checked} glossary terms against docs/{args.language}.") - if unused: - print(f"\n{len(unused)} prescribed but unused — something else took over:\n") - for source, target, uses in unused: - print(f" {source} -> {target} (English {uses}×, translation never)") - return 1 - print("Every prescribed term is in use.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/check_typography.py b/scripts/check_typography.py deleted file mode 100644 index d8a4980..0000000 --- a/scripts/check_typography.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -"""Count the typography rules a translation has to obey, per language. - -Written after two naive greps produced only false positives: searching for the -character pair »…« in Norwegian matches the gap *between* two correct «…» pairs, -and searching for a space before a colon matches English comments inside code -fences. Both looked like defects and neither was one. - -So quotations are checked by walking the marks in order and requiring them to -alternate open, close, open, close — which is what "the pairs are the right way -round" actually means — and everything is measured with code fences, inline code -and admonition syntax removed first. -""" - -from __future__ import annotations - -import re -import sys -from pathlib import Path - -# Which mark opens a quotation, and which closes it, per language. -QUOTES = { - "fi": ("”", "”"), # ”…” — the same character opens and closes -} -# French is the one language that *requires* a space before ; : ! ? — and -# requires it to be unbreakable, so the line never breaks before the mark. -# Everywhere else any space there is an error, which is why this cannot be one -# rule for all: applying the French habit elsewhere is a known leak, and -# applying the majority rule to French would flag every correct sentence. -SPACE_REQUIRED = {"fr"} -PLAIN_SPACE_BEFORE_PUNCT = re.compile(r"\u0020[;:!?]") -# German compounds a multi-word proper name with hyphens throughout — -# NMEA-2000-Netzwerk, Signal-K-Server — and its glossary calls a missing hyphen -# there the most visible marker of a translation done by someone who does not -# write German. Every other language treats that same chain as an error, and a -# hyphen at the *junction* between a product name and a common noun -# (HaLOS-avbilder) is right in the Germanic languages and wrong in the Romance -# ones. One rule cannot serve all three cases, so each is scoped to where its -# glossary asks for it. -HYPHEN_CHAINS = re.compile(r"NMEA-2000|Signal-K|Raspberry-Pi|Compute-Module") -CHAINS_ALLOWED = {"de"} -JUNCTION_HYPHEN = re.compile( - r"\b(?:HALPI2|HaLOS|NMEA 2000|Signal K|Raspberry Pi|E7T)-" - r"[a-z\u00e1\u00e9\u00ed\u00f3\u00fa\u00f1\u00e0\u00e8\u00ec\u00f2\u00f9]" -) -JUNCTION_FORBIDDEN = {"es", "it"} -SPACE_BEFORE_PUNCT = re.compile(r"[   ][;:!?]") - - -def prose(text: str) -> str: - """The text a reader sees, with everything that is markup taken out. - - Inline code becomes a placeholder rather than nothing: deleting it joins the - words on either side and manufactures a space before the next punctuation - mark, which is exactly the false positive this function exists to avoid. - """ - text = re.sub(r"^---\n.*?\n---\n", "", text, flags=re.S) - text = re.sub(r"```.*?```", "\n", text, flags=re.S) - text = re.sub(r"`[^`\n]*`", "X", text) - text = re.sub(r'^!!! \w+ ".*"$', "", text, flags=re.M) # admonition syntax quotes - text = re.sub(r"\]\([^)]*\)", "]", text) # link targets - # A table's delimiter row carries the column alignment as colons — | ---: | - # — which reads as a space before a colon and is not prose at all. - text = re.sub(r"^[|\s:-]+$", "", text, flags=re.M) - # Repository names and filenames are identifiers that happen to contain - # hyphens — HALPI2-hardware, HALPI2-schematic_v0.6.1.pdf — and reading them - # as compounds of the target language invents defects that are not there. - text = re.sub(r"https?://\S+", "X", text) - text = re.sub( - r"\b[\w.-]+\.(?:pdf|zip|png|jpe?g|md|txt|json|ya?ml|step|bin|conf|sock)\b", - "X", - text, - ) - return text - - -def quotation_faults(text: str, opening: str, closing: str) -> list[str]: - """Marks must alternate open, close, open, close — and end closed.""" - if opening == closing: - count = text.count(opening) - return [] if count % 2 == 0 else [f"odd number of {opening} ({count})"] - faults, depth = [], 0 - for index, char in enumerate(text): - if char == opening: - if depth: - faults.append( - f"{opening} opens while already open: " - f"...{text[max(0, index - 40) : index + 20]}..." - ) - depth += 1 - elif char == closing: - if not depth: - faults.append( - f"{closing} closes nothing: " - f"...{text[max(0, index - 40) : index + 20]}..." - ) - else: - depth -= 1 - if depth: - faults.append(f"{depth} quotation(s) never closed") - return faults - - -def main() -> int: - languages = sys.argv[1:] or sorted(QUOTES) - worst = 0 - for language in languages: - opening, closing = QUOTES[language] - pages = sorted(Path("docs", language).rglob("*.md")) - quotes = spacing = chains = 0 - problems: list[str] = [] - for page in pages: - text = prose(page.read_text(encoding="utf-8")) - for fault in quotation_faults(text, opening, closing): - quotes += 1 - problems.append(f" {page}: {fault}") - rule = ( - PLAIN_SPACE_BEFORE_PUNCT - if language in SPACE_REQUIRED - else SPACE_BEFORE_PUNCT - ) - for match in rule.finditer(text): - spacing += 1 - wrong = "breakable space" if language in SPACE_REQUIRED else "space" - problems.append( - f" {page}: {wrong} before '{match.group()[-1]}': " - f"...{text[max(0, match.start() - 40):match.end() + 10]}..." - ) - allowed = language in CHAINS_ALLOWED - chain_rule = () if allowed else HYPHEN_CHAINS.finditer(text) - for match in chain_rule: - chains += 1 - problems.append( - f" {page}: hyphen inside a product name '{match.group()}'" - ) - if language in JUNCTION_FORBIDDEN: - for match in JUNCTION_HYPHEN.finditer(text): - chains += 1 - problems.append( - f" {page}: junction hyphen '{match.group()}' " - f"— not used in this language" - ) - - marks = sum(prose(p.read_text(encoding="utf-8")).count(opening) for p in pages) - status = "ok" if not problems else f"{len(problems)} PROBLEMS" - print( - f"{language}: {len(pages)} pages, {marks} quotations " - f"({opening}…{closing}), quote faults {quotes}, spacing {spacing}, " - f"hyphen chains {chains} — {status}" - ) - for problem in problems[:8]: - print(problem) - worst = max(worst, len(problems)) - return 1 if worst else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/map_anchors.py b/scripts/map_anchors.py deleted file mode 100644 index 7166e34..0000000 --- a/scripts/map_anchors.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 -"""Rewrite English anchor fragments in a translation to the translated slugs. - -Anchor slugs come from heading text, so a translated heading gets a different -slug and every link pointing at it breaks — including links on pages nobody -touched. Translators leave the English fragment in place; this maps it across. - -The mapping is positional: the structure comparison already proves the -translation has the same headings in the same order, so the nth heading of the -English page and the nth heading of the translation are the same heading. That -is stronger than matching on text, which cannot work once the text is in another -language. - -Usage: map_anchors.py [--apply] -Without --apply it only reports what it would change. -""" - -from __future__ import annotations - -import re -import sys -from pathlib import Path - -HEADING_ID = re.compile(r" list[str]: - """Heading ids of a built page, in document order. - - The default language has no URL segment of its own — `docs/en/index.md` is - served at the site root, not under `en/` — so English pages are looked up - without a prefix. - """ - stem = page[: -len(".md")] - stem = "" if stem == "index" else stem.removesuffix("/index") - prefix = "" if language == "en" else language - parts = [p for p in (prefix, stem) if p] - html = site.joinpath(*parts, "index.html") - if not html.exists(): - raise SystemExit( - f"No built page for {language}/{page} at {html} — build the site first." - ) - return HEADING_ID.findall(html.read_text(encoding="utf-8")) - - -def target_page(link: str, page: str) -> str | None: - """The markdown page a link points at, relative to the docs root.""" - path, _, _ = link.partition("#") - if link.startswith(("http://", "https://", "mailto:")): - return None - if not path: - return page - resolved = (Path(page).parent / path).as_posix() - resolved = Path(resolved).resolve().relative_to(Path.cwd().resolve()).as_posix() - return resolved if resolved.endswith(".md") else None - - -def main() -> int: - site, language = Path(sys.argv[1]), sys.argv[2] - apply = "--apply" in sys.argv - docs = Path("docs") - - english = { - p.relative_to(docs / "en").as_posix(): built_ids( - site, "en", p.relative_to(docs / "en").as_posix() - ) - for p in (docs / "en").rglob("*.md") - } - translated = {page: built_ids(site, language, page) for page in english} - - changes, unmapped = [], [] - for page in sorted(english): - source = docs / language / page - if not source.exists(): - continue - text = original = source.read_text(encoding="utf-8") - for link in set(LINK.findall(text)): - path, _, fragment = link.partition("#") - target = target_page(link, page) - if target is None or target not in english: - continue - ids_en, ids_tr = english[target], translated[target] - if fragment not in ids_en: - continue - if len(ids_en) != len(ids_tr): - unmapped.append( - f"{language}/{page} -> {link}: {target} has " - f"{len(ids_en)} headings in English, {len(ids_tr)} translated" - ) - continue - replacement = ids_tr[ids_en.index(fragment)] - if replacement != fragment: - text = text.replace(f"]({link})", f"]({path}#{replacement})") - changes.append( - f" {language}/{page}\n {fragment} -> {replacement}" - ) - if text != original: - if apply: - source.write_text(text, encoding="utf-8") - - verb = "rewritten" if apply else "to rewrite" - print(f"{len(changes)} anchors {verb} in docs/{language}.") - for change in changes: - print(change) - if unmapped: - print(f"\n{len(unmapped)} could not be mapped — structure differs:") - for problem in unmapped: - print(f" {problem}") - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/stamp_translation.py b/scripts/stamp_translation.py deleted file mode 100644 index 61fc83c..0000000 --- a/scripts/stamp_translation.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 -"""Write the translated_from stamp into a translation's frontmatter. - -Stamp a translation only when it has actually been (re-)translated against the -current English source. A stamp updated without real translation work reports -green and makes the staleness invisible — that is the one gap the status check -cannot close. - - uv run python scripts/stamp_translation.py docs/fi/user-guide/hardware.md -""" - -from __future__ import annotations - -import argparse -import subprocess -import sys -from pathlib import Path - -from translation_status import configured_languages - -DOCS = Path("docs") -STAMP_KEY = "translated_from" - - -def english_source(translation: Path, default: str) -> Path: - """docs// -> docs//.""" - parts = translation.parts - if len(parts) < 3 or parts[0] != DOCS.name: - raise SystemExit(f"{translation}: not a path under docs//") - if parts[1] == default: - raise SystemExit( - f"{translation}: this is a source page, not a translation. " - f"Source pages carry no stamp — that is the point: an English edit " - f"needs no ceremony." - ) - return DOCS / default / Path(*parts[2:]) - - -def blob_hash(path: Path) -> str: - return subprocess.run( - ["git", "hash-object", str(path)], - capture_output=True, text=True, check=True, - ).stdout.strip() - - -def restamp(text: str, value: str) -> str: - """Set the stamp, replacing an existing one and preserving other keys.""" - line = f"{STAMP_KEY}: {value}" - if not text.startswith("---\n"): - return f"---\n{line}\n---\n\n{text}" - end = text.find("\n---", 4) - if end == -1: - raise SystemExit("frontmatter is not terminated") - front, body = text[4:end], text[end + 4:].lstrip("\n") - kept = [l for l in front.splitlines() if not l.startswith(f"{STAMP_KEY}:")] - return "---\n" + "\n".join([*kept, line]) + "\n---\n\n" + body - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("translations", nargs="+", type=Path) - args = parser.parse_args() - - default, _ = configured_languages() - for translation in args.translations: - if not translation.exists(): - raise SystemExit(f"{translation}: does not exist") - source = english_source(translation, default) - if not source.exists(): - raise SystemExit(f"{translation}: no English source at {source}") - value = blob_hash(source) - translation.write_text( - restamp(translation.read_text(encoding="utf-8"), value), - encoding="utf-8", - ) - print(f"{translation}: {STAMP_KEY} = {value}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/translation_status.py b/scripts/translation_status.py deleted file mode 100644 index ff22ea3..0000000 --- a/scripts/translation_status.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -"""Report which translations are missing or out of date. - -A translation records the git blob hash of the English source it was written -against, in its own frontmatter: - - --- - translated_from: at translation time> - --- - -The English page carries nothing, so an English edit needs no ceremony: editing -it changes its content, which changes its hash, which makes every translation of -it report as stale on its own. - -Reports; never blocks. Exit status is 0 unless the check itself could not run. -""" - -from __future__ import annotations - -import argparse -import subprocess -import sys -import tempfile -from dataclasses import dataclass -from pathlib import Path - -import yaml - -DOCS = Path("docs") -STAMP_KEY = "translated_from" - - -class _Loader(yaml.SafeLoader): - """mkdocs.yml carries python/name tags that SafeLoader refuses to parse.""" - - -_Loader.add_multi_constructor("", lambda loader, suffix, node: None) - - -def configured_languages() -> tuple[str, list[str]]: - """Return (default language, other languages) from the i18n plugin config.""" - config = yaml.load(Path("mkdocs.yml").read_text(encoding="utf-8"), Loader=_Loader) - for plugin in config.get("plugins", []): - if isinstance(plugin, dict) and "i18n" in plugin: - languages = plugin["i18n"]["languages"] - default = next(l["locale"] for l in languages if l.get("default")) - others = [l["locale"] for l in languages if not l.get("default")] - return default, others - raise SystemExit("mkdocs.yml has no i18n plugin configuration") - - -def blob_hash(path: Path) -> str: - return subprocess.run( - ["git", "hash-object", str(path)], - capture_output=True, text=True, check=True, - ).stdout.strip() - - -def stamp_of(path: Path) -> str | None: - """Read translated_from from a page's frontmatter, if it has one.""" - text = path.read_text(encoding="utf-8") - if not text.startswith("---\n"): - return None - end = text.find("\n---", 4) - if end == -1: - return None - front = yaml.safe_load(text[4:end]) or {} - value = front.get(STAMP_KEY) - return str(value) if value else None - - -def english_diff(stamped: str, current: Path) -> str | None: - """Diff the stamped English blob against the English page as it stands now. - - The current page is compared from the working tree rather than as a stored - object: `git hash-object` computes a hash without writing the object, so - diffing two hashes would fail on the side that was never stored. - """ - blob = subprocess.run( - ["git", "cat-file", "-p", stamped], capture_output=True, text=True, - ) - if blob.returncode != 0: - return None # stamped blob not in this clone — CI needs fetch-depth: 0 - with tempfile.TemporaryDirectory() as tmp: - was = Path(tmp) / current.name - was.write_text(blob.stdout, encoding="utf-8") - result = subprocess.run( - ["git", "diff", "--no-index", "--no-color", str(was), str(current)], - capture_output=True, text=True, - ) - # --no-index exits 1 when the files differ, which is the expected case. - # Drop the file headers: they carry a temporary path, and the page is - # already named in the surrounding report. - noise = ("diff --git ", "index ", "--- ", "+++ ") - return "\n".join( - line for line in result.stdout.splitlines() - if not line.startswith(noise) - ) - - -@dataclass -class Entry: - language: str - page: str # path relative to the language directory - state: str # missing | unstamped | stale | orphaned | current - expected: str # blob hash the translation should record - diff: str | None = None - - -def collect(default: str, languages: list[str], want_diff: bool) -> list[Entry]: - sources = sorted(p for p in (DOCS / default).rglob("*.md")) - entries: list[Entry] = [] - for source in sources: - relative = source.relative_to(DOCS / default) - expected = blob_hash(source) - for language in languages: - target = DOCS / language / relative - if not target.exists(): - entries.append(Entry(language, str(relative), "missing", expected)) - continue - stamped = stamp_of(target) - if stamped is None: - entries.append(Entry(language, str(relative), "unstamped", expected)) - elif stamped == expected: - entries.append(Entry(language, str(relative), "current", expected)) - else: - diff = english_diff(stamped, source) if want_diff else None - entries.append(Entry(language, str(relative), "stale", expected, diff)) - - # A translation whose source was deleted is invisible to the loop above, - # because that walks the sources. It is still a page being served. - for language in languages: - root = DOCS / language - for translation in sorted(root.rglob("*.md")): - if not (DOCS / default / translation.relative_to(root)).exists(): - entries.append( - Entry(language, str(translation.relative_to(root)), "orphaned", "") - ) - return entries - - -def render_text(entries: list[Entry]) -> str: - out = [] - for language in sorted({e.language for e in entries}): - rows = [e for e in entries if e.language == language] - counts = {s: sum(1 for e in rows if e.state == s) for s in - ("current", "stale", "unstamped", "missing", "orphaned")} - out.append(f"{language}: " + " ".join(f"{k}={v}" for k, v in counts.items())) - for entry in rows: - if entry.state != "current": - out.append(f" {entry.state:9s} {entry.page}") - if entry.expected: - out.append(f" {STAMP_KEY}: {entry.expected}") - return "\n".join(out) - - -def render_markdown(entries: list[Entry], only: set[str] | None) -> str: - shown = [e for e in entries if only is None or e.page in only] - out = ["## Translation status", ""] - for language in sorted({e.language for e in entries}): - rows = [e for e in entries if e.language == language] - counts = {s: sum(1 for e in rows if e.state == s) for s in - ("current", "stale", "unstamped", "missing", "orphaned")} - summary = ", ".join(f"{v} {k}" for k, v in counts.items() if v) - out.append(f"**{language}** — {summary}") - out.append("") - - behind = [e for e in shown if e.state != "current"] - if not behind: - out.append("Every translation of the pages in scope is current.") - return "\n".join(out) - - out += ["| Language | Page | State | Stamp to record |", - "|:---|:---|:---|:---|"] - for entry in behind: - out.append(f"| {entry.language} | `{entry.page}` | {entry.state} | `{entry.expected}` |") - out.append("") - - for entry in behind: - if entry.diff: - out += [f"
English changes since " - f"{entry.language}/{entry.page} was translated", - "", "```diff", entry.diff.rstrip(), "```", "", "
", ""] - elif entry.state == "stale": - out.append(f"") - return "\n".join(out) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--format", choices=("text", "markdown"), default="text") - parser.add_argument("--diff", action="store_true", - help="include the English diff for stale pages") - parser.add_argument("--only-pages", nargs="*", metavar="PATH", - help="restrict the detail section to these docs//-relative paths") - args = parser.parse_args() - - default, languages = configured_languages() - if not languages: - print("No translation languages configured.") - return 0 - - entries = collect(default, languages, want_diff=args.diff) - if args.format == "markdown": - only = set(args.only_pages) if args.only_pages else None - print(render_markdown(entries, only)) - else: - print(render_text(entries)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/uv.lock b/uv.lock index 30ea860..224725f 100644 --- a/uv.lock +++ b/uv.lock @@ -156,6 +156,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, ] +[[package]] +name = "halos-docs-tools" +version = "0.1.0" +source = { git = "https://github.com/halos-org/docs-tools?rev=v0.1.0#7f09d05f54cf64184a7a4d7205c166bc49ec0f6e" } +dependencies = [ + { name = "pyyaml" }, +] + [[package]] name = "idna" version = "3.11" @@ -508,6 +516,7 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "click" }, + { name = "halos-docs-tools" }, { name = "mkdocs-material" }, { name = "mkdocs-static-i18n" }, ] @@ -515,6 +524,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "click", specifier = "<8.3" }, + { name = "halos-docs-tools", git = "https://github.com/halos-org/docs-tools?rev=v0.1.0" }, { name = "mkdocs-material", specifier = ">=9.5" }, { name = "mkdocs-static-i18n", specifier = ">=1.2" }, ] From efce09beb908de794955b4cdf904eb2037ca9787 Mon Sep 17 00:00:00 2001 From: Matti Airas Date: Thu, 13 Aug 2026 18:16:58 +0300 Subject: [PATCH 2/2] fix(docs): close the gaps review found in the adoption The documented pre-flight could not reproduce the gate. The Verifying block ran `translation-status`, which always exits 0; CI runs `translation-status --check`. A translator following the skill got a clean local run and a red check -- the same false green this migration exists to remove. deploy.yml holds pages: write and id-token: write and ran plain `uv sync`, while the gate runs `uv sync --locked`. Now that a git dependency is in the graph, bumping the pin without re-locking made the two disagree: the gate refuses, the deploy resolves the new ref live and runs its build backend with the strongest token in the repository. The skill now says where a new locale gets registered: the glossary and typography registries are hardcoded in the package, so a tenth locale needs an entry in each, a release, and a pin bump. Also: the caller stanza states what the called workflow enforces; the Verifying block said four and listed five; and the language-selector note named two repositories and a count of three where four carry the block. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/skills/translate-page/SKILL.md | 16 +++++++++++----- .github/workflows/deploy.yml | 2 +- .github/workflows/translation-status.yml | 4 ++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.claude/skills/translate-page/SKILL.md b/.claude/skills/translate-page/SKILL.md index e6ffec6..af300ad 100644 --- a/.claude/skills/translate-page/SKILL.md +++ b/.claude/skills/translate-page/SKILL.md @@ -90,6 +90,12 @@ fixing a typo), the English source did not change: leave the stamp alone. ## Adding a language to the site +`check-glossary` and `check-typography` accept a fixed set of locales, and those +registries live in the `halos-docs-tools` package, not in this repository. A new +locale needs an entry in each, a release of that package, and a bump of the pin +in `pyproject.toml`. Until that lands both commands reject the locale, while +`translation-status` reads `mkdocs.yml` and starts failing the gate immediately. + When a locale is added to `mkdocs.yml`, check the language selector too. The Material theme caps the open menu at `10rem`, which fits five entries at the site's font size; the sixth language onward scrolls out of sight behind a @@ -105,9 +111,9 @@ scrollbar that gives no hint anything is below it. ``` 24rem clears thirteen entries; the viewport term keeps the menu on screen on a -short display. The same block is in the HALPI2 and HALMET repositories — keep -the three identical, and add it to any further site that gains a second -language. +short display. The same block is in the HALPI2, HALMET, SH-RPi and SH-ESP32 +repositories — keep the four identical, and add it to any further site +that gains a second language. Verify by measuring rather than by eye: open the site, read the rule's `max-height` off the stylesheet, and compare it against the list's natural @@ -117,12 +123,12 @@ is captured. ## Verifying -All four, every time: +All five, every time: ```bash uv run mkdocs build --strict uv run check-anchors site -uv run translation-status +uv run translation-status --check uv run check-glossary fi uv run check-typography fi ``` diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 91aba16..195acaa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 - - run: uv sync + - run: uv sync --locked - run: uv run mkdocs build --strict - uses: actions/upload-pages-artifact@v3 with: diff --git a/.github/workflows/translation-status.yml b/.github/workflows/translation-status.yml index 7ddedbf..fdccf5d 100644 --- a/.github/workflows/translation-status.yml +++ b/.github/workflows/translation-status.yml @@ -24,5 +24,9 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + # The called workflow builds the site, checks its anchors, and fails the run + # when any translation is stale, missing, unstamped or orphaned. It judges the + # whole repository, not the diff, so an edit to an English page needs its + # translations re-stamped in the same pull request. translation-status: uses: halos-org/shared-workflows/.github/workflows/translation-status.yml@main