diff --git a/.github/workflows/translate.yml b/.github/workflows/translate.yml index 803be26..4a302a0 100644 --- a/.github/workflows/translate.yml +++ b/.github/workflows/translate.yml @@ -21,8 +21,12 @@ on: permissions: contents: write pull-requests: write - # Lets the built-in GITHUB_TOKEN call GitHub Models. No API key, no billing. - models: read + # No 'models: read' here on purpose. GitHub Models access is a Copilot + # entitlement, and GITHUB_TOKEN carries the ORGANISATION's entitlement rather + # than the triggering user's. This org has no Copilot plan, so the built-in + # token gets 403 whatever the permissions say. Translation therefore + # authenticates with the MODELS_TOKEN secret; GITHUB_TOKEN still does the + # git and pull request work below. jobs: translate: @@ -42,7 +46,12 @@ jobs: - name: Translate missing strings id: translate env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Whichever of these is set decides the provider. OPENROUTER_API_KEY + # is preferred; MODELS_TOKEN is a personal access token with + # models:read, needed because the org has no Copilot plan and the + # built-in GITHUB_TOKEN therefore cannot reach GitHub Models. + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + MODELS_TOKEN: ${{ secrets.MODELS_TOKEN }} run: | python3 scripts/translate.py \ ${{ inputs.languages && format('--languages {0}', inputs.languages) || '' }} @@ -73,8 +82,8 @@ jobs: title: 'chore: update translations' body: | `translations/en.json` changed, so the strings that were missing from - each language have been machine-translated using GitHub Models. - Existing translations were not re-sent to the model. + each language have been machine-translated. Existing translations + were not re-sent to the model. ## Changes diff --git a/README.md b/README.md index a1569b8..7a82b13 100644 --- a/README.md +++ b/README.md @@ -109,15 +109,48 @@ plainly wrong phrasing. Corrections are very welcome, and they stick: edit. Once you have corrected a string it is treated as yours. If the English source later changes, the workflow flags the string for review rather than replacing your version. -- Only strings that are missing, or whose English source was reworded, are ever - sent to a model. -To add a language, add its code and name to `LANGUAGES` in -`scripts/translate.py` and open a pull request; the workflow fills in the file. +One style note if you are correcting a string: translations deliberately avoid +the familiar/polite distinction (German du/Sie, French tu/vous, and so on) by +using impersonal phrasing, such as infinitives for instructions. Please keep +that style. -Translations deliberately avoid the familiar/polite distinction (German du/Sie, -French tu/vous, and so on) by using impersonal phrasing such as infinitives for -instructions. Please keep that style when correcting a string. +Missing a language? Open an issue and we will add it. + +
+Maintaining the translations (developer notes) + +`scripts/translate.py` fills in strings that are missing from a language, or +whose English source was reworded since it was last translated. Nothing else is +ever sent to a model. `.github/workflows/translate.yml` runs it when +`translations/en.json` changes on a release branch and opens a pull request. + +**Adding a language.** Add its code and name to `LANGUAGES` in +`scripts/translate.py`. The next run fills in the file. + +**Providers.** Any OpenAI-compatible chat endpoint works. OpenRouter and GitHub +Models are configured out of the box in `PROVIDERS`, selected by whichever API +key is present: + +| Variable | Provider | Notes | +|---|---|---| +| `OPENROUTER_API_KEY` | OpenRouter | Preferred. No output-token cap. | +| `MODELS_TOKEN` | GitHub Models | Personal access token with `models:read`. | +| `GITHUB_TOKEN` | GitHub Models | Only works if the repo's org has a Copilot plan. | + +`TRANSLATE_PROVIDER` and `TRANSLATE_MODEL` override the choice for one run: + +```bash +OPENROUTER_API_KEY=... TRANSLATE_MODEL=google/gemini-2.5-flash-lite \ + python3 scripts/translate.py --languages de --dry-run +``` + +**Checking output.** `scripts/verify_translations.py` re-checks the files on +disk and fails on placeholder mismatches, empty values, or keys that no longer +exist in `en.json`. It also warns when a translation addresses the reader +directly, which the impersonal style above is meant to avoid. + +
## Contributing - Feature requests and bug reports are welcome! Please open an issue on GitHub diff --git a/scripts/translate.py b/scripts/translate.py index 9d26cae..d9b0ead 100755 --- a/scripts/translate.py +++ b/scripts/translate.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 -"""Fill in missing Home Assistant translations using GitHub Models. +"""Fill in missing Home Assistant translations with an LLM. + +Works with any OpenAI-compatible chat/completions endpoint. OpenRouter and +GitHub Models are configured out of the box; whichever API key is present in +the environment decides which is used. See PROVIDERS. Source of truth is ``translations/en.json`` -- the fully resolved English file. ``strings.json`` is NOT usable here: it contains ``[%key:...%]`` references that @@ -66,19 +70,64 @@ def _ssl_context() -> ssl.SSLContext: # True to let fresh machine output win instead. OVERWRITE_MANUAL_EDITS = False -API_URL = "https://models.github.ai/inference/chat/completions" +# Both providers speak the same OpenAI-compatible chat/completions shape, so +# switching is a matter of URL, key and model. The provider is chosen by +# whichever key is present, or forced with TRANSLATE_PROVIDER. +PROVIDERS = { + # Credit-based. No per-request output cap, and not tied to anyone's Copilot + # entitlement, which is why it is tried first. + "openrouter": { + "url": "https://openrouter.ai/api/v1/chat/completions", + "model": "openai/gpt-4.1-mini", + "keys": ("OPENROUTER_API_KEY",), + }, + # Free, but access is a Copilot entitlement: a repository GITHUB_TOKEN + # carries the ORGANISATION's entitlement, so an org with no Copilot plan + # gets 403 regardless of workflow permissions. MODELS_TOKEN (a personal + # access token with models:read) is the way around that. + "github": { + "url": "https://models.github.ai/inference/chat/completions", + "model": "openai/gpt-4.1-mini", + "keys": ("MODELS_TOKEN", "GITHUB_TOKEN"), + }, +} + + +def resolve_provider() -> tuple[str, dict, str]: + """Pick a provider from the environment. Returns (name, config, key).""" + forced = os.environ.get("TRANSLATE_PROVIDER") + if forced: + if forced not in PROVIDERS: + raise SystemExit( + f"TRANSLATE_PROVIDER={forced!r} is not one of " + f"{', '.join(PROVIDERS)}" + ) + candidates = [(forced, PROVIDERS[forced])] + else: + candidates = list(PROVIDERS.items()) + + override = os.environ.get("TRANSLATE_MODEL") + for name, config in candidates: + for variable in config["keys"]: + key = os.environ.get(variable) + if key: + if override: + config = {**config, "model": override} + return name, config, key + + name, config = candidates[0] + if override: + config = {**config, "model": override} + return name, config, "" -# Low rate-limit tier: 150 requests/day vs 50 for high tier. Tagged multilingual. -# Verify against https://models.github.ai/catalog/models before changing. -MODEL = "openai/gpt-4.1-mini" # GitHub Models caps output at 4000 tokens per request on the Copilot Free/Pro -# tier, well below the model's own 32k ceiling. That cap -- not the model -- is -# why we chunk. ~80 strings lands around 1800 output tokens, leaving margin for -# languages that render longer than English. +# tier, far below the model's own ceiling. OpenRouter has no such cap, but the +# chunk size is kept for both: ~80 strings lands near 1800 output tokens, which +# also bounds how much work a single malformed response can cost. CHUNK_SIZE = 80 -# 15 requests/minute on the low tier. +# GitHub Models allows 15 requests/minute on the low tier. Harmless elsewhere. MIN_SECONDS_BETWEEN_REQUESTS = 4.5 LANGUAGES = { @@ -254,11 +303,45 @@ def write_translation_file(path: Path, flat: dict[str, str]) -> None: ) -def call_model(token: str, language: str, batch: dict[str, str]) -> dict[str, str]: +class ApiError(RuntimeError): + """A non-2xx response from the models endpoint, with the body preserved.""" + + def __init__(self, status: int, detail: str) -> None: + self.status = status + self.detail = detail + super().__init__(f"HTTP {status}: {detail or '(no response body)'}") + + def advice(self) -> str: + """Actionable next step for the failures that actually happen.""" + if self.status in (401, 403): + return ( + "The token was rejected for GitHub Models.\n" + " Access to GitHub Models is a Copilot entitlement. A\n" + " repository GITHUB_TOKEN carries the ORGANISATION's\n" + " entitlement, not that of whoever triggered the run, so an\n" + " org with no Copilot plan gets 403 however the workflow\n" + " permissions are set. That is why this repository uses a\n" + " MODELS_TOKEN secret instead.\n" + " Check that MODELS_TOKEN is set, unexpired, and carries the\n" + " models:read scope. Locally, export it in your shell." + ) + if self.status == 404: + return ( + f"Model not found at this provider. Check the id against the " + "provider's model catalog." + ) + if self.status >= 500: + return "The models endpoint failed. Re-run the workflow." + return "" + + +def call_model( + api: dict, token: str, language: str, batch: dict[str, str] +) -> dict[str, str]: """Translate one batch of strings. Raises on transport or parse failure.""" body = json.dumps( { - "model": MODEL, + "model": api["model"], "temperature": 0, "response_format": {"type": "json_object"}, "messages": [ @@ -279,7 +362,7 @@ def call_model(token: str, language: str, batch: dict[str, str]) -> dict[str, st ).encode("utf-8") request = urllib.request.Request( - API_URL, + api["url"], data=body, headers={ "Authorization": f"Bearer {token}", @@ -288,8 +371,19 @@ def call_model(token: str, language: str, batch: dict[str, str]) -> dict[str, st }, ) - with urllib.request.urlopen(request, timeout=180, context=SSL_CONTEXT) as response: - payload = json.load(response) + try: + with urllib.request.urlopen( + request, timeout=180, context=SSL_CONTEXT + ) as response: + payload = json.load(response) + except urllib.error.HTTPError as err: + # The API explains itself in the response body. Surfacing it turns an + # opaque "HTTP Error 403: Forbidden" traceback into something callable. + try: + detail = err.read().decode("utf-8", "replace").strip()[:400] + except Exception: # noqa: BLE001 - diagnostics must never mask the error + detail = "" + raise ApiError(err.code, detail) from err content = payload["choices"][0]["message"]["content"].strip() @@ -334,6 +428,7 @@ def validate( def translate_language( + api: dict, token: str, code: str, language: str, @@ -361,9 +456,9 @@ def translate_language( throttle[0] = time.monotonic() try: - response = call_model(token, language, numbered) - except urllib.error.HTTPError as err: - if err.code == 429: + response = call_model(api, token, language, numbered) + except ApiError as err: + if err.status == 429: # Daily or per-minute budget exhausted. Keep what we have: a # partial-but-valid PR beats a failed run. print(f" [{code}] rate limited, stopping early", file=sys.stderr) @@ -386,8 +481,8 @@ def translate_language( time.sleep(MIN_SECONDS_BETWEEN_REQUESTS) throttle[0] = time.monotonic() try: - retry = call_model(token, language, {"0": todo[key]}) - except urllib.error.HTTPError: + retry = call_model(api, token, language, {"0": todo[key]}) + except ApiError: rejected[key] = reason continue retried = {key: retry["0"]} if isinstance(retry.get("0"), str) else {} @@ -446,16 +541,26 @@ def main() -> int: else: codes = list(LANGUAGES) - token = os.environ.get("GITHUB_TOKEN", "") + # MODELS_TOKEN first: GitHub Models access is a Copilot entitlement, and a + # repository GITHUB_TOKEN carries the organisation's entitlement. An org + # without a Copilot plan gets 403 no matter what the workflow permissions + # say, so a personal token has to stand in. + provider, api, token = resolve_provider() if not token and not args.dry_run: print( - "GITHUB_TOKEN is not set. In Actions, add 'models: read' to the " - "workflow permissions block. Locally, export a PAT with the " - "models:read scope.", + "No API key found. Set one of: " + + ", ".join(v for c in PROVIDERS.values() for v in c["keys"]) + + ".\n" + "GITHUB_TOKEN only works when the repository's organisation has a " + "Copilot plan; otherwise use OPENROUTER_API_KEY or a MODELS_TOKEN " + "personal access token with the models:read scope.", file=sys.stderr, ) return 1 + if not args.dry_run: + print(f"provider: {provider} ({api['model']})") + state: dict[str, dict[str, dict[str, str]]] = {} if STATE_FILE.exists(): state = json.loads(STATE_FILE.read_text(encoding="utf-8")) @@ -501,7 +606,7 @@ def main() -> int: print(f"{code} ({language}): translating {len(todo)} strings") accepted, rejected = ( - translate_language(token, code, language, todo, throttle) + translate_language(api, token, code, language, todo, throttle) if todo else ({}, {}) ) @@ -560,4 +665,10 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + try: + sys.exit(main()) + except ApiError as error: + print(f"\n{error}", file=sys.stderr) + if guidance := error.advice(): + print(f"\n{guidance}", file=sys.stderr) + sys.exit(1)