From 50885d3ba4cce11a8702b69685539816b7dd1cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henrique=20Ara=C3=BAjo?= Date: Fri, 23 Oct 2026 19:47:11 -0300 Subject: [PATCH] feat: Tolgee-compatible translation endpoints Code written for Tolgee's REST API keeps working after a move to LowLex with only a new base URL, project and API key: - POST /v2/projects/{project}/translations creates the key when missing and sets its texts; an empty or null text clears a translation. - PUT on the same path updates existing keys only (404 otherwise). - GET /v2/projects/{project}/translations/{languages} returns {"en": {...}, ...}, nested on dots, with ns, structureDelimiter and filterTag as in Tolgee. {project} is a LowLex slug or id; without it the API key's project is used. Usual project permissions apply, texts get the "api" origin, and an unchanged text keeps its reviewed state. /v2/ answers CORS like /api/. --- README.md | 10 + internal/server/server.go | 4 +- internal/server/tolgee_api.go | 295 +++++++++++++++++++++++++++++ internal/server/tolgee_api_test.go | 92 +++++++++ 4 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 internal/server/tolgee_api.go create mode 100644 internal/server/tolgee_api_test.go diff --git a/README.md b/README.md index ada4fd8..35bf113 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,16 @@ Import fields (multipart): `file`, `lang` (otherwise from the file name), `names The import is a background task: it reads 500 keys at a time, shows its progress under **Tasks** and on the project page, and saves its position after every page. If LowLex restarts, it continues where it stopped; a failed or cancelled import can be retried from the same point. A 160,000-key project imports in seconds once Tolgee has answered. +After the move, code written for Tolgee's REST API keeps working. Point it at LowLex, use a LowLex API key (`lx_…`, write scope to change texts) and put the project slug (or its LowLex id) where the Tolgee project id was: + +| Tolgee endpoint | What LowLex does | +|---|---| +| `POST /v2/projects/{project}/translations` | Sets `{"key", "namespace", "translations": {"en": "…"}}`, creating the key when it does not exist. An empty or `null` text clears that translation. | +| `PUT /v2/projects/{project}/translations` | Same, but the key must already exist (404 otherwise). | +| `GET /v2/projects/{project}/translations/{languages}` | `{"en": {…}, "pt-BR": {…}}` for a comma list of languages. Keys nest on dots unless `structureDelimiter=` is empty; `ns` picks a namespace (default: none) and `filterTag` keeps tagged keys. | + +The same paths without `{project}` (`/v2/projects/translations`) use the API key's project. Texts written this way show **api** as their origin, and sending a text that has not changed keeps its reviewed state. + You can also export from Tolgee as JSON with one file per language (ZIP) and import the ZIP; that keeps keys, languages and namespaces, and review states start as *translated*. ### Background tasks diff --git a/internal/server/server.go b/internal/server/server.go index 297351f..f595058 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -130,7 +130,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.Set("Referrer-Policy", "same-origin") h.Set("X-Frame-Options", "DENY") - if strings.HasPrefix(r.URL.Path, "/api/") { + if strings.HasPrefix(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/v2/") { // API keys are meant for apps and CI, including browser apps on other origins. // Cookies are never sent with a wildcard origin, so sessions stay same-origin. h.Set("Access-Control-Allow-Origin", "*") @@ -215,6 +215,8 @@ func (s *Server) routes() { m.Handle("POST /api/projects/{slug}/api-keys", s.project(roleDeveloper, s.createAPIKey)) m.Handle("DELETE /api/projects/{slug}/api-keys/{id}", s.project(roleDeveloper, s.deleteAPIKey)) + s.tolgeeRoutes() + m.Handle("/api/", handler(func(w http.ResponseWriter, r *http.Request) error { return errNotFound })) m.HandleFunc("/", s.serveUI) } diff --git a/internal/server/tolgee_api.go b/internal/server/tolgee_api.go new file mode 100644 index 0000000..ba1d8cd --- /dev/null +++ b/internal/server/tolgee_api.go @@ -0,0 +1,295 @@ +package server + +import ( + "bufio" + "database/sql" + "encoding/json" + "errors" + "net/http" + "sort" + "strconv" + "strings" + + "github.com/LowBloat/LowLex/internal/formats" +) + +// A subset of Tolgee's REST API, so code written for Tolgee keeps working after a move to +// LowLex with only a new base URL, project and API key. {projectId} is a LowLex project slug +// or id; without it, the project comes from the API key, as in Tolgee. + +func (s *Server) tolgeeRoutes() { + m := s.mux + for _, base := range []string{"/v2/projects/{projectId}/translations", "/v2/projects/translations"} { + m.Handle("GET "+base+"/{languages}", s.tolgeeProject(roleViewer, s.tolgeeGetTranslations)) + m.Handle("POST "+base, s.tolgeeProject(roleTranslator, s.tolgeeSetTranslations(true))) + m.Handle("PUT "+base, s.tolgeeProject(roleTranslator, s.tolgeeSetTranslations(false))) + } + m.Handle("/v2/", handler(func(w http.ResponseWriter, r *http.Request) error { return errNotFound })) +} + +// tolgeeProject finds the project slug, then hands over to the usual project checks. +func (s *Server) tolgeeProject(minRole int, h func(w http.ResponseWriter, r *http.Request, pc *projectCtx) error) http.Handler { + inner := s.project(minRole, h) + return handler(func(w http.ResponseWriter, r *http.Request) error { + slug := r.PathValue("projectId") + if slug == "" { + p, err := s.authenticate(r) + if err != nil { + return err + } + if p.apiKey == nil { + return badRequest("Name the project in the path: /v2/projects/{project}/translations") + } + if err := s.db.QueryRow(`SELECT slug FROM projects WHERE id = ?`, p.apiKey.projectID).Scan(&slug); err != nil { + return err + } + } else if id, err := strconv.ParseInt(slug, 10, 64); err == nil { + // a slug wins over an id that looks the same + s.db.QueryRow(`SELECT slug FROM projects WHERE id = ? AND NOT EXISTS (SELECT 1 FROM projects WHERE slug = ?)`, id, slug).Scan(&slug) + } + r.SetPathValue("slug", slug) + inner.ServeHTTP(w, r) + return nil + }) +} + +type tolgeeTranslation struct { + ID int64 `json:"id"` + Text *string `json:"text"` + State string `json:"state"` + Outdated bool `json:"outdated"` + Auto bool `json:"auto"` + MtProvider *string `json:"mtProvider"` +} + +// tolgeeSetTranslations is Tolgee's POST (create the key when missing) and PUT (the key must +// exist). An empty or null text clears that translation; an unchanged text keeps its state. +func (s *Server) tolgeeSetTranslations(create bool) func(w http.ResponseWriter, r *http.Request, pc *projectCtx) error { + return func(w http.ResponseWriter, r *http.Request, pc *projectCtx) error { + var in struct { + Key string `json:"key"` + Namespace *string `json:"namespace"` + Translations map[string]*string `json:"translations"` + LanguagesToReturn []string `json:"languagesToReturn"` + } + if err := readJSON(r, &in); err != nil { + return err + } + name, err := cleanKeyName(in.Key) + if err != nil { + return err + } + ns := "" + if in.Namespace != nil { + ns = strings.TrimSpace(*in.Namespace) + } + codes := make([]string, 0, len(in.Translations)) + for code := range in.Translations { + if !pc.canEditLang(code) { + return errf(http.StatusForbidden, "language_forbidden", "You cannot edit %s in this project", code) + } + codes = append(codes, code) + } + sort.Strings(codes) + + tx, err := s.db.BeginTx(r.Context(), nil) + if err != nil { + return err + } + defer tx.Rollback() + var keyID int64 + created := false + err = tx.QueryRow(`SELECT id FROM keys WHERE project_id = ? AND namespace = ? AND name = ?`, pc.ID, ns, name).Scan(&keyID) + switch { + case errors.Is(err, sql.ErrNoRows) && !create: + return errf(http.StatusNotFound, "key_not_found", "The key %q does not exist", name) + case errors.Is(err, sql.ErrNoRows): + if pc.role < roleDeveloper { + return errForbidden + } + res, err := tx.Exec(`INSERT INTO keys (project_id, namespace, name) VALUES (?, ?, ?)`, pc.ID, ns, name) + if err != nil { + return err + } + keyID, _ = res.LastInsertId() + created = true + case err != nil: + return err + } + + ts := now() + var changed []string + for _, code := range codes { + var lid int64 + if err := tx.QueryRow(`SELECT id FROM languages WHERE project_id = ? AND code = ?`, pc.ID, code).Scan(&lid); err != nil { + return badRequest("The project has no language %q", code) + } + var cur, state string + err := tx.QueryRow(`SELECT text, state FROM translations WHERE key_id = ? AND language_id = ?`, keyID, lid).Scan(&cur, &state) + exists := err == nil + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + text := "" + if in.Translations[code] != nil { + text = *in.Translations[code] + } + if (exists && text == cur) || (!exists && text == "") { + continue + } + if exists && state == "reviewed" && pc.role < roleReviewer { + return errf(http.StatusForbidden, "reviewed_locked", "Only reviewers can change a reviewed translation") + } + if text == "" { + _, err = tx.Exec(`DELETE FROM translations WHERE key_id = ? AND language_id = ?`, keyID, lid) + } else { + _, err = tx.Exec(`INSERT INTO translations (key_id, language_id, text, state, origin, updated_by, updated_at) VALUES (?, ?, ?, 'translated', 'api', ?, ?) + ON CONFLICT (key_id, language_id) DO UPDATE SET text = excluded.text, state = excluded.state, origin = excluded.origin, + updated_by = excluded.updated_by, updated_at = excluded.updated_at`, keyID, lid, text, nullID(pc.userID), ts) + } + if err != nil { + return err + } + changed = append(changed, code) + } + if len(changed) > 0 && !created { + if _, err := tx.Exec(`UPDATE keys SET updated_at = ? WHERE id = ?`, ts, keyID); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + if created { + s.logActivity(pc.ID, pc.userID, "key.create", name) + } + if len(changed) > 0 { + s.logActivity(pc.ID, pc.userID, "translation.set", name+" ["+strings.Join(changed, ", ")+"]") + } + + ret := in.LanguagesToReturn + if ret == nil { + ret = codes + } + out := map[string]tolgeeTranslation{} + for _, code := range ret { + t := tolgeeTranslation{State: "UNTRANSLATED"} + var text, state, origin string + err := s.db.QueryRow(`SELECT t.rowid, t.text, t.state, t.origin FROM translations t JOIN languages l ON l.id = t.language_id + WHERE t.key_id = ? AND l.project_id = ? AND l.code = ?`, keyID, pc.ID, code).Scan(&t.ID, &text, &state, &origin) + if err == nil { + t.Text, t.State = &text, "TRANSLATED" + if state == "reviewed" { + t.State = "REVIEWED" + } + if kind, provider, ok := strings.Cut(origin, ":"); ok && (kind == "mt" || kind == "ai") { + t.Auto, t.MtProvider = true, &provider + } + } else if !errors.Is(err, sql.ErrNoRows) { + return err + } + out[code] = t + } + var keyNS *string + if ns != "" { + keyNS = &ns + } + writeJSON(w, 200, map[string]any{"keyId": keyID, "keyName": name, "keyNamespace": keyNS, "keyIsPlural": false, "translations": out}) + return nil + } +} + +// tolgeeGetTranslations returns {"en": {...}, "pt-BR": {...}} for comma-separated languages. +// Like Tolgee: ns picks one namespace (default: the empty one), keys nest on dots unless +// structureDelimiter is empty, and filterTag keeps keys that have any of the given tags. +func (s *Server) tolgeeGetTranslations(w http.ResponseWriter, r *http.Request, pc *projectCtx) error { + q := r.URL.Query() + format := "json-nested" + if d, ok := q["structureDelimiter"]; ok { + switch d[0] { + case "": + format = "json" + case ".": + default: + return badRequest("structureDelimiter must be \".\" or empty") + } + } + langs, err := s.projectLanguages(r.Context(), pc.ID) + if err != nil { + return err + } + known := map[string]bool{} + for _, l := range langs { + known[l.Code] = true + } + var codes []string + for _, c := range strings.Split(r.PathValue("languages"), ",") { + if c = strings.TrimSpace(c); c == "" { + continue + } + if !known[c] { + return errf(http.StatusNotFound, "language_not_found", "The project has no language %q", c) + } + codes = append(codes, c) + } + + keys, err := s.exportKeys(r.Context(), pc.ID, true, q.Get("ns")) + if err != nil { + return err + } + if tags := q["filterTag"]; len(tags) > 0 { + tagged := map[int64]bool{} + args := []any{pc.ID} + for _, t := range tags { + args = append(args, t) + } + rows, err := s.db.QueryContext(r.Context(), `SELECT DISTINCT k.id FROM keys k, json_each(k.tags) j + WHERE k.project_id = ? AND j.value IN (?`+strings.Repeat(", ?", len(tags)-1)+`)`, args...) + if err != nil { + return err + } + for rows.Next() { + var id int64 + rows.Scan(&id) + tagged[id] = true + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + kept := keys[:0] + for _, k := range keys { + if tagged[k.id] { + kept = append(kept, k) + } + } + keys = kept + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + bw := bufio.NewWriter(w) + bw.WriteByte('{') + for i, code := range codes { + texts, err := s.exportTexts(r.Context(), pc.ID, code, "") + if err != nil { + return err + } + entries := []formats.Entry{} + for _, k := range keys { + if text, ok := texts[k.id]; ok { + entries = append(entries, formats.Entry{Key: k.name, Text: text}) + } + } + if i > 0 { + bw.WriteByte(',') + } + name, _ := json.Marshal(code) + bw.Write(name) + bw.WriteByte(':') + if err := formats.Write(bw, format, entries); err != nil { + return err + } + } + bw.WriteByte('}') + return bw.Flush() +} diff --git a/internal/server/tolgee_api_test.go b/internal/server/tolgee_api_test.go new file mode 100644 index 0000000..31631ac --- /dev/null +++ b/internal/server/tolgee_api_test.go @@ -0,0 +1,92 @@ +package server + +import ( + "encoding/json" + "fmt" + "testing" +) + +// Calls shaped like an app written for Tolgee's REST API. +func TestTolgeeCompatibleAPI(t *testing.T) { + _, newClient := newTestServer(t) + admin := newClient() + admin.do("POST", "/api/auth/login", map[string]string{"username": "admin", "password": DefaultAdminPassword}, 200) + admin.do("POST", "/api/auth/password", map[string]string{"current_password": DefaultAdminPassword, "new_password": "a-much-better-password"}, 200) + admin.do("POST", "/api/projects", map[string]any{"name": "Platify", "base_language": "en", "languages": []string{"pt-BR"}}, 201) + admin.do("POST", "/api/projects", map[string]any{"name": "Other"}, 201) + projectID := int(admin.do("GET", "/api/projects/platify", nil, 200)["project"].(map[string]any)["id"].(float64)) + + app := newClient() + app.key = admin.do("POST", "/api/projects/platify/api-keys", map[string]string{"name": "backend", "scope": "write"}, 201)["key"].(string) + reader := newClient() + reader.key = admin.do("POST", "/api/projects/platify/api-keys", map[string]string{"name": "front", "scope": "read"}, 201)["key"].(string) + + // POST creates the key, even with only an empty placeholder + res := app.do("POST", "/v2/projects/platify/translations", map[string]any{"key": "game_1_summary", "translations": map[string]string{"en": ""}}, 200) + if res["keyName"] != "game_1_summary" || res["keyId"].(float64) == 0 { + t.Fatalf("create: %v", res) + } + if tr := res["translations"].(map[string]any)["en"].(map[string]any); tr["text"] != nil || tr["state"] != "UNTRANSLATED" { + t.Fatalf("placeholder: %v", tr) + } + + // then fills it in, by slug, by numeric id and with the project taken from the key + app.do("POST", "/v2/projects/platify/translations", map[string]any{"key": "game_1_summary", "translations": map[string]string{"en": "A game"}}, 200) + app.do("POST", fmt.Sprintf("/v2/projects/%d/translations", projectID), map[string]any{"key": "nav.home", "translations": map[string]string{"en": "Home", "pt-BR": "Início"}}, 200) + res = app.do("POST", "/v2/projects/translations", map[string]any{"key": "game_1_summary", "translations": map[string]string{"pt-BR": "Um jogo"}, "languagesToReturn": []string{"en", "pt-BR"}}, 200) + trs := res["translations"].(map[string]any) + if trs["en"].(map[string]any)["text"] != "A game" || trs["pt-BR"].(map[string]any)["state"] != "TRANSLATED" { + t.Fatalf("languagesToReturn: %v", trs) + } + keys := admin.do("GET", "/api/projects/platify/keys", nil, 200) + if keys["total"].(float64) != 2 { + t.Fatalf("keys: %v", keys) + } + + // PUT only updates existing keys + app.do("PUT", "/v2/projects/platify/translations", map[string]any{"key": "nope", "translations": map[string]string{"en": "x"}}, 404) + app.do("PUT", "/v2/projects/platify/translations", map[string]any{"key": "nav.home", "translations": map[string]string{"en": "Start"}}, 200) + + // a reviewed text sent again unchanged stays reviewed; an empty one clears it + k := admin.do("GET", "/api/projects/platify/keys?q=nav.home", nil, 200)["keys"].([]any)[0].(map[string]any) + admin.do("PUT", fmt.Sprintf("/api/projects/platify/translations/%d/pt-BR", int(k["id"].(float64))), map[string]any{"state": "reviewed"}, 200) + res = app.do("POST", "/v2/projects/platify/translations", map[string]any{"key": "nav.home", "translations": map[string]string{"pt-BR": "Início"}}, 200) + if st := res["translations"].(map[string]any)["pt-BR"].(map[string]any)["state"]; st != "REVIEWED" { + t.Fatalf("unchanged reviewed text: %v", st) + } + app.do("POST", "/v2/projects/platify/translations", map[string]any{"key": "game_1_summary", "translations": map[string]any{"pt-BR": nil}}, 200) + + // permissions and bad input + reader.do("POST", "/v2/projects/platify/translations", map[string]any{"key": "x", "translations": map[string]string{"en": "x"}}, 403) + app.do("POST", "/v2/projects/other/translations", map[string]any{"key": "x", "translations": map[string]string{"en": "x"}}, 403) + app.do("POST", "/v2/projects/platify/translations", map[string]any{"key": "x", "translations": map[string]string{"de": "x"}}, 400) + app.do("POST", "/v2/projects/platify/translations", map[string]any{"key": " ", "translations": map[string]string{"en": "x"}}, 400) + admin.do("POST", "/v2/projects/translations", map[string]any{"key": "x", "translations": map[string]string{"en": "x"}}, 400) + newClient().do("POST", "/v2/projects/platify/translations", map[string]any{"key": "x"}, 401) + app.do("GET", "/v2/projects/platify/nothing", nil, 404) + + // GET returns every language at once, nested by default + code, body := reader.raw("/v2/projects/platify/translations/en,pt-BR") + var all map[string]map[string]any + if err := json.Unmarshal([]byte(body), &all); code != 200 || err != nil { + t.Fatalf("get: %d %s", code, body) + } + if all["en"]["nav"].(map[string]any)["home"] != "Start" || all["en"]["game_1_summary"] != "A game" { + t.Fatalf("nested en: %s", body) + } + if _, ok := all["pt-BR"]["game_1_summary"]; ok { + t.Fatalf("cleared text still exported: %s", body) + } + _, body = reader.raw("/v2/projects/translations/pt-BR?structureDelimiter=") + var flat map[string]map[string]string + if json.Unmarshal([]byte(body), &flat) != nil || flat["pt-BR"]["nav.home"] != "Início" { + t.Fatalf("flat: %s", body) + } + _, body = reader.raw("/v2/projects/platify/translations/en?ns=other-ns") + if err := json.Unmarshal([]byte(body), &all); err != nil || len(all["en"]) != 0 { + t.Fatalf("namespace filter: %s", body) + } + if code, _ := reader.raw("/v2/projects/platify/translations/de"); code != 404 { + t.Fatalf("unknown language: %d", code) + } +}