From 83ac36392663bf6d5eea49693509edebf3b9c490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henrique=20Ara=C3=BAjo?= Date: Fri, 23 Oct 2026 11:06:23 -0300 Subject: [PATCH] feat: import from Tolgee through its API, background tasks with progress - Background job queue in SQLite with two workers. Each batch saves progress and its resume point in the same transaction; jobs left running at shutdown resume on the next start, failed or cancelled ones can be retried. - Tolgee import: project API key or personal token, languages, namespaces, descriptions, tags, translated and reviewed texts, 500 keys per page with cursor paging and retries. Verified against a real Tolgee server. - Machine translation of whole languages now runs as a job. - Tasks page, progress strip on the project page, progress in the batch dialog. - Exports stream per language (ZIP and CSV written to the response), so large projects stay well under the container memory limit. - Load test: 160,000 keys import in about 8 s. --- README.md | 11 +- cmd/lowlex/main.go | 4 +- internal/formats/formats.go | 40 +- internal/server/io.go | 169 +++++--- internal/server/jobs.go | 363 ++++++++++++++++++ internal/server/load_test.go | 70 ++++ internal/server/providers.go | 227 +++++++---- internal/server/server.go | 10 + internal/server/server_test.go | 25 +- internal/server/tolgee_import.go | 333 ++++++++++++++++ internal/server/tolgee_test.go | 169 ++++++++ internal/store/schema2.sql | 25 ++ internal/store/store.go | 5 +- internal/tolgee/tolgee.go | 261 +++++++++++++ web/src/lib/api.ts | 30 ++ .../components/BatchTranslateDialog.svelte | 56 ++- .../lib/components/TolgeeImportDialog.svelte | 150 ++++++++ web/src/lib/jobs.svelte.ts | 71 ++++ web/src/lib/locales/en.json | 47 +++ web/src/lib/locales/pt-BR.json | 47 +++ web/src/routes/(app)/+layout.svelte | 5 +- web/src/routes/(app)/jobs/+page.svelte | 121 ++++++ web/src/routes/(app)/p/[slug]/+layout.svelte | 71 +++- web/src/routes/(app)/projects/+page.svelte | 20 +- 24 files changed, 2147 insertions(+), 183 deletions(-) create mode 100644 internal/server/jobs.go create mode 100644 internal/server/load_test.go create mode 100644 internal/server/tolgee_import.go create mode 100644 internal/server/tolgee_test.go create mode 100644 internal/store/schema2.sql create mode 100644 internal/tolgee/tolgee.go create mode 100644 web/src/lib/components/TolgeeImportDialog.svelte create mode 100644 web/src/lib/jobs.svelte.ts create mode 100644 web/src/routes/(app)/jobs/+page.svelte diff --git a/README.md b/README.md index 8efa8d6..ada4fd8 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ LowLex keeps the keys and translations of your apps in one place, lets developer - **Translation editor.** Search keys, texts and context; filter by missing, draft, not reviewed or reviewed per language; pick which language columns to show; edit every language of a key side by side; `Ctrl + Enter` saves. - **Machine translation and AI.** Google Cloud Translation, DeepL, Microsoft Translator, LibreTranslate and any OpenAI-compatible chat API (OpenAI, OpenRouter, Groq, Ollama, LM Studio, vLLM…). Ask every provider for a suggestion on one text, or fill a whole language in one go as drafts. AI providers get the project description, the key name and its context, keep placeholders (`{name}`, `{{count}}`, `%s`, ICU plurals, tags) and can follow your own instructions or glossary. Provider keys are encrypted at rest (AES-256-GCM). - **Import and export.** JSON (flat or nested, i18next / vue-i18n style), CSV with every language in one sheet, Gettext PO, Android `strings.xml`, Apple `.strings` and Java `.properties`. Upload one file or a ZIP of many: the language comes from the file name and folders become namespaces. +- **Straight from Tolgee.** Import a whole Tolgee project through its API in a background task with progress and resume, see [Coming from Tolgee](#coming-from-tolgee). - **API keys per project.** Read keys for apps and builds, write keys for CI. Every endpoint the UI uses also takes an API key. - **Activity history** per project. @@ -120,7 +121,15 @@ Import fields (multipart): `file`, `lang` (otherwise from the file name), `names ### Coming from Tolgee -In Tolgee, export the project as JSON with one file per language (ZIP), then import the ZIP into a LowLex project. Keys, languages and namespaces come across; review states start as *translated*. +**Projects → Import from Tolgee** reads a project straight from your Tolgee server through its API, so even very large projects move without exporting files. Give it the Tolgee URL and a project API key (`tgpak_…`, scope `translations.view`) or a personal access token (`tgpat_…`) plus the project id. LowLex brings over languages, namespaces, key descriptions, tags, translated texts and the reviewed state. + +The import is a background task: it reads 500 keys at a time, shows its progress under **Tasks** and on the project page, and saves its position after every page. If LowLex restarts, it continues where it stopped; a failed or cancelled import can be retried from the same point. A 160,000-key project imports in seconds once Tolgee has answered. + +You can also export from Tolgee as JSON with one file per language (ZIP) and import the ZIP; that keeps keys, languages and namespaces, and review states start as *translated*. + +### Background tasks + +Tolgee imports and machine translation of whole languages run as tasks under **Tasks**, with progress, an estimate of the time left, stop and retry. They survive restarts. ## Develop diff --git a/cmd/lowlex/main.go b/cmd/lowlex/main.go index c8b94f3..19d6da2 100644 --- a/cmd/lowlex/main.go +++ b/cmd/lowlex/main.go @@ -102,9 +102,11 @@ func serve(cfg config.Config) error { } addr := net.JoinHostPort(cfg.Host, strconv.Itoa(cfg.Port)) + app := server.New(db, cfg, box, ui) + defer app.Close() // running background jobs go back to the queue srv := &http.Server{ Addr: addr, - Handler: server.New(db, cfg, box, ui), + Handler: app, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 2 * time.Minute, } diff --git a/internal/formats/formats.go b/internal/formats/formats.go index d3c900f..74a0f02 100644 --- a/internal/formats/formats.go +++ b/internal/formats/formats.go @@ -12,6 +12,7 @@ import ( "encoding/xml" "errors" "fmt" + "io" "regexp" "sort" "strconv" @@ -252,21 +253,42 @@ func ParseCSV(data []byte) (map[string]map[string]string, map[string]string, err return texts, descs, nil } +// CSVWriter streams the all-languages sheet row by row. +type CSVWriter struct { + w *csv.Writer + langs []string +} + +// NewCSVWriter writes the BOM (so spreadsheet apps read UTF-8) and the header row. +func NewCSVWriter(out io.Writer, langs []string) *CSVWriter { + io.WriteString(out, "\xef\xbb\xbf") + c := &CSVWriter{w: csv.NewWriter(out), langs: langs} + c.w.Write(append([]string{"key", "description"}, langs...)) + return c +} + +func (c *CSVWriter) Write(key, description string, texts map[string]string) error { + rec := []string{key, description} + for _, l := range c.langs { + rec = append(rec, texts[l]) + } + return c.w.Write(rec) +} + +func (c *CSVWriter) Flush() error { + c.w.Flush() + return c.w.Error() +} + // RenderCSV writes every language in one sheet. rows[key][lang] = text. func RenderCSV(keys []Entry, langs []string, rows map[string]map[string]string) []byte { var buf bytes.Buffer - buf.WriteString("\xef\xbb\xbf") // BOM so spreadsheet apps read UTF-8 - w := csv.NewWriter(&buf) - w.Write(append([]string{"key", "description"}, langs...)) + cw := NewCSVWriter(&buf, langs) sort.Slice(keys, func(i, j int) bool { return keys[i].Key < keys[j].Key }) for _, k := range keys { - rec := []string{k.Key, k.Description} - for _, l := range langs { - rec = append(rec, rows[k.Key][l]) - } - w.Write(rec) + cw.Write(k.Key, k.Description, rows[k.Key]) } - w.Flush() + cw.Flush() return buf.Bytes() } diff --git a/internal/server/io.go b/internal/server/io.go index 8687495..53052f1 100644 --- a/internal/server/io.go +++ b/internal/server/io.go @@ -371,30 +371,31 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx) return badRequest("Unknown language in lang=%s", q.Get("lang")) } - data, err := s.exportData(r.Context(), pc, nsFilter, ns, minState) + keys, err := s.exportKeys(r.Context(), pc.ID, nsFilter, ns) if err != nil { return err } base := pc.Slug if format == "csv" { - entries := data.allKeys() - rows := make(map[string]map[string]string, len(entries)) - for i, k := range data.keys { - rows[entries[i].Key] = data.texts[k.id] - } - body := formats.RenderCSV(entries, codes, rows) - return sendFile(w, base+".csv", "text/csv; charset=utf-8", body) + return s.exportCSV(w, r, pc.ID, keys, codes, minState, nsFilter, ns, base+".csv") } - render := func(code, namespace string) ([]byte, error) { + // Texts are read one language at a time, so a large project never sits in memory whole. + var baseTexts map[int64]string + if fallback { + if baseTexts, err = s.exportTexts(r.Context(), pc.ID, pc.BaseLanguage, minState); err != nil { + return err + } + } + render := func(texts map[int64]string, namespace string) ([]byte, error) { entries := []formats.Entry{} - for _, k := range data.keys { + for _, k := range keys { if k.ns != namespace { continue } - text, ok := data.texts[k.id][code] + text, ok := texts[k.id] if !ok && fallback { - text, ok = data.texts[k.id][pc.BaseLanguage] + text, ok = baseTexts[k.id] } if !ok { continue @@ -403,25 +404,35 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx) } return formats.Render(format, entries) } - namespaces := data.namespaces() + namespaces := keyNamespaces(keys) if len(codes) == 1 && len(namespaces) <= 1 { nsName := "" if len(namespaces) == 1 { nsName = namespaces[0] } - body, err := render(codes[0], nsName) + texts, err := s.exportTexts(r.Context(), pc.ID, codes[0], minState) + if err != nil { + return err + } + body, err := render(texts, nsName) if err != nil { return err } return sendFile(w, exportName(info, codes[0], ""), contentType(info.Ext), body) } - var buf bytes.Buffer - zw := zip.NewWriter(&buf) + // the ZIP is written straight to the response + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Disposition", `attachment; filename="`+base+"-"+format+`.zip"`) + zw := zip.NewWriter(w) for _, code := range codes { + texts, err := s.exportTexts(r.Context(), pc.ID, code, minState) + if err != nil { + return err + } for _, n := range namespaces { - body, err := render(code, n) + body, err := render(texts, n) if err != nil { return err } @@ -429,13 +440,12 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx) if err != nil { return err } - f.Write(body) + if _, err := f.Write(body); err != nil { + return err + } } } - if err := zw.Close(); err != nil { - return err - } - return sendFile(w, base+"-"+format+".zip", "application/zip", buf.Bytes()) + return zw.Close() } func exportName(info formats.Info, code, ns string) string { @@ -478,15 +488,10 @@ type exportKey struct { ns, name, desc string } -type exportSet struct { - keys []exportKey - texts map[int64]map[string]string -} - -func (e *exportSet) namespaces() []string { +func keyNamespaces(keys []exportKey) []string { seen := map[string]bool{} out := []string{} - for _, k := range e.keys { + for _, k := range keys { if !seen[k.ns] { seen[k.ns] = true out = append(out, k.ns) @@ -495,21 +500,9 @@ func (e *exportSet) namespaces() []string { return out } -func (e *exportSet) allKeys() []formats.Entry { - out := make([]formats.Entry, len(e.keys)) - for i, k := range e.keys { - name := k.name - if k.ns != "" { - name = k.ns + ":" + k.name - } - out[i] = formats.Entry{Key: name, Description: k.desc} - } - return out -} - -func (s *Server) exportData(ctx context.Context, pc *projectCtx, nsFilter bool, ns, minState string) (*exportSet, error) { +func (s *Server) exportKeys(ctx context.Context, projectID int64, nsFilter bool, ns string) ([]exportKey, error) { q := `SELECT id, namespace, name, description FROM keys WHERE project_id = ?` - args := []any{pc.ID} + args := []any{projectID} if nsFilter { q += ` AND namespace = ?` args = append(args, ns) @@ -518,42 +511,98 @@ func (s *Server) exportData(ctx context.Context, pc *projectCtx, nsFilter bool, if err != nil { return nil, err } - set := &exportSet{texts: map[int64]map[string]string{}} + defer rows.Close() + keys := []exportKey{} for rows.Next() { var k exportKey if err := rows.Scan(&k.id, &k.ns, &k.name, &k.desc); err != nil { - rows.Close() return nil, err } - set.keys = append(set.keys, k) + keys = append(keys, k) } - rows.Close() + return keys, rows.Err() +} - tq := `SELECT t.key_id, l.code, t.text FROM translations t JOIN languages l ON l.id = t.language_id - JOIN keys k ON k.id = t.key_id WHERE k.project_id = ?` +func stateFilter(minState string) string { switch minState { case "translated": - tq += ` AND t.state IN ('translated', 'reviewed')` + return ` AND t.state IN ('translated', 'reviewed')` case "reviewed": - tq += ` AND t.state = 'reviewed'` + return ` AND t.state = 'reviewed'` } - trows, err := s.db.QueryContext(ctx, tq, pc.ID) + return "" +} + +// exportTexts returns one language's texts by key id. +func (s *Server) exportTexts(ctx context.Context, projectID int64, code, minState string) (map[int64]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT t.key_id, t.text FROM translations t JOIN languages l ON l.id = t.language_id + WHERE l.project_id = ? AND l.code = ?`+stateFilter(minState), projectID, code) if err != nil { return nil, err } - defer trows.Close() - for trows.Next() { + defer rows.Close() + texts := map[int64]string{} + for rows.Next() { var id int64 - var code, text string - if err := trows.Scan(&id, &code, &text); err != nil { + var text string + if err := rows.Scan(&id, &text); err != nil { return nil, err } - if set.texts[id] == nil { - set.texts[id] = map[string]string{} + texts[id] = text + } + return texts, rows.Err() +} + +// exportCSV streams every language side by side, reading translations in key order. +func (s *Server) exportCSV(w http.ResponseWriter, r *http.Request, projectID int64, keys []exportKey, codes []string, minState string, nsFilter bool, ns, name string) error { + // same keys and same order as exportKeys, so the two lists can be merged + q := `SELECT t.key_id, l.code, t.text FROM translations t + JOIN languages l ON l.id = t.language_id JOIN keys k ON k.id = t.key_id + WHERE k.project_id = ?` + stateFilter(minState) + args := []any{projectID} + if nsFilter { + q += ` AND k.namespace = ?` + args = append(args, ns) + } + rows, err := s.db.QueryContext(r.Context(), q+` ORDER BY k.namespace, k.name`, args...) + if err != nil { + return err + } + defer rows.Close() + w.Header().Set("Content-Type", "text/csv; charset=utf-8") + w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`) + cw := formats.NewCSVWriter(w, codes) + + var ( + rowKey int64 = -1 + rowLang string + rowText string + more = rows.Next() + ) + if more { + if err := rows.Scan(&rowKey, &rowLang, &rowText); err != nil { + return err + } + } + for _, k := range keys { + texts := map[string]string{} + for more && rowKey == k.id { + texts[rowLang] = rowText + if more = rows.Next(); more { + if err := rows.Scan(&rowKey, &rowLang, &rowText); err != nil { + return err + } + } + } + display := k.name + if k.ns != "" { + display = k.ns + ":" + k.name + } + if err := cw.Write(display, k.desc, texts); err != nil { + return err } - set.texts[id][code] = text } - return set, trows.Err() + return cw.Flush() } // --- API keys --- diff --git a/internal/server/jobs.go b/internal/server/jobs.go new file mode 100644 index 0000000..4027b0e --- /dev/null +++ b/internal/server/jobs.go @@ -0,0 +1,363 @@ +package server + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "strconv" + "sync" + "time" +) + +// Background jobs: long work (a Tolgee import, machine translation of a whole language) runs in +// a worker goroutine and saves its progress and resume point after every batch. A job that was +// running when the server stopped goes back to the queue and continues from that point. + +const jobWorkers = 2 + +var errJobCancelled = errors.New("cancelled") + +type jobHandler func(ctx context.Context, j *jobRun) (result any, err error) + +// jobHandlers maps a job kind to its code; filled in by the files that define each kind. +var jobHandlers = map[string]jobHandler{} + +type jobRunner struct { + wake chan struct{} + stop context.CancelFunc + wg sync.WaitGroup + mu sync.Mutex + running map[int64]context.CancelFunc +} + +// jobRun is a claimed job as its handler sees it. +type jobRun struct { + s *Server + ID int64 + ProjectID int64 + Kind string + UserID int64 + Params []byte + State string + Total int + Done int + lastSave time.Time +} + +func (s *Server) startJobs() { + ctx, cancel := context.WithCancel(context.Background()) + s.jobs = &jobRunner{wake: make(chan struct{}, 1), stop: cancel, running: map[int64]context.CancelFunc{}} + // jobs left running by a previous process resume from their saved state + s.db.Exec(`UPDATE jobs SET status = 'queued' WHERE status = 'running'`) + for range jobWorkers { + s.jobs.wg.Add(1) + go s.jobWorker(ctx) + } +} + +// Close stops the job workers; running jobs are put back in the queue. +func (s *Server) Close() { + if s.jobs == nil { + return + } + s.jobs.stop() + s.jobs.wg.Wait() +} + +func (s *Server) wakeJobs() { + select { + case s.jobs.wake <- struct{}{}: + default: + } +} + +func (s *Server) jobWorker(ctx context.Context) { + defer s.jobs.wg.Done() + for { + j, err := s.claimJob() + if err != nil { + log.Printf("jobs: %v", err) + } + if j == nil { + select { + case <-ctx.Done(): + return + case <-s.jobs.wake: + case <-time.After(5 * time.Second): + } + continue + } + s.runJob(ctx, j) + if ctx.Err() != nil { + return + } + } +} + +func (s *Server) claimJob() (*jobRun, error) { + j := &jobRun{s: s} + var sealed string + ts := now() + err := s.db.QueryRow(`UPDATE jobs SET status = 'running', started_at = COALESCE(started_at, ?), updated_at = ? + WHERE id = (SELECT id FROM jobs WHERE status = 'queued' ORDER BY id LIMIT 1) + RETURNING id, COALESCE(project_id, 0), kind, params, state, total, done, COALESCE(created_by, 0)`, ts, ts). + Scan(&j.ID, &j.ProjectID, &j.Kind, &sealed, &j.State, &j.Total, &j.Done, &j.UserID) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if sealed != "" { + if j.Params, err = s.box.Open(sealed); err != nil { + s.finishJob(j.ID, "failed", "Cannot read the job parameters (was LOWLEX_SECRET changed?)", nil) + return nil, nil + } + } + return j, nil +} + +func (s *Server) runJob(parent context.Context, j *jobRun) { + ctx, cancel := context.WithCancel(parent) + s.jobs.mu.Lock() + s.jobs.running[j.ID] = cancel + s.jobs.mu.Unlock() + defer func() { + cancel() + s.jobs.mu.Lock() + delete(s.jobs.running, j.ID) + s.jobs.mu.Unlock() + }() + + h := jobHandlers[j.Kind] + if h == nil { + s.finishJob(j.ID, "failed", "Unknown job kind "+j.Kind, nil) + return + } + var result any + err := func() (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("internal error: %v", r) + } + }() + result, err = h(ctx, j) + return err + }() + switch { + case err == nil: + s.finishJob(j.ID, "done", "", result) + case errors.Is(err, errJobCancelled): + s.finishJob(j.ID, "cancelled", "", result) + case parent.Err() != nil: + // server shutting down: leave it for the next start + s.db.Exec(`UPDATE jobs SET status = 'queued', updated_at = ? WHERE id = ?`, now(), j.ID) + default: + log.Printf("job %d (%s) failed: %v", j.ID, j.Kind, err) + s.finishJob(j.ID, "failed", err.Error(), result) + } +} + +func (s *Server) finishJob(id int64, status, message string, result any) { + res := "" + if result != nil { + b, _ := json.Marshal(result) + res = string(b) + } + ts := now() + s.db.Exec(`UPDATE jobs SET status = ?, message = ?, result = ?, finished_at = ?, updated_at = ? WHERE id = ?`, + status, truncate(message, 2000), res, ts, ts, id) +} + +// SetTotal records how many items the job will process. +func (j *jobRun) SetTotal(total int) { + j.Total = total + j.s.db.Exec(`UPDATE jobs SET total = ?, updated_at = ? WHERE id = ?`, total, now(), j.ID) +} + +// Note sets the job's status line without changing its progress. +func (j *jobRun) Note(msg string) { + j.s.db.Exec(`UPDATE jobs SET message = ?, updated_at = ? WHERE id = ?`, truncate(msg, 500), now(), j.ID) +} + +// Checkpoint saves progress and the resume state inside the batch's transaction, so the saved +// state always matches what is in the database. It reports errJobCancelled when asked to stop. +func (j *jobRun) Checkpoint(tx *sql.Tx, done int, state string) error { + j.Done, j.State = done, state + var cancel bool + if err := tx.QueryRow(`UPDATE jobs SET done = ?, state = ?, message = '', updated_at = ? WHERE id = ? RETURNING cancel_requested`, + done, state, now(), j.ID).Scan(&cancel); err != nil { + return err + } + if cancel { + return errJobCancelled + } + return nil +} + +// Cancelled reports whether someone asked to stop this job. +func (j *jobRun) Cancelled() bool { + var c bool + j.s.db.QueryRow(`SELECT cancel_requested FROM jobs WHERE id = ?`, j.ID).Scan(&c) + return c +} + +func (s *Server) createJob(projectID int64, kind, title string, userID int64, params any) (int64, error) { + plain, err := json.Marshal(params) + if err != nil { + return 0, err + } + res, err := s.db.Exec(`INSERT INTO jobs (project_id, kind, title, params, created_by) VALUES (?, ?, ?, ?, ?)`, + nullID(projectID), kind, title, s.box.Seal(plain), nullID(userID)) + if err != nil { + return 0, err + } + id, _ := res.LastInsertId() + s.wakeJobs() + return id, nil +} + +// --- API --- + +type jobView struct { + ID int64 `json:"id"` + Kind string `json:"kind"` + Title string `json:"title"` + Status string `json:"status"` + Total int `json:"total"` + Done int `json:"done"` + Message string `json:"message"` + Result json.RawMessage `json:"result,omitempty"` + ProjectSlug string `json:"project_slug,omitempty"` + ProjectName string `json:"project_name,omitempty"` + CreatedBy string `json:"created_by"` + CreatedAt string `json:"created_at"` + StartedAt *string `json:"started_at"` + FinishedAt *string `json:"finished_at"` + UpdatedAt string `json:"updated_at"` + Cancelling bool `json:"cancelling"` +} + +const jobSelect = `SELECT j.id, j.kind, j.title, j.status, j.total, j.done, j.message, j.result, + COALESCE(p.slug, ''), COALESCE(p.name, ''), COALESCE(u.username, ''), j.created_at, j.started_at, j.finished_at, j.updated_at, j.cancel_requested + FROM jobs j LEFT JOIN projects p ON p.id = j.project_id LEFT JOIN users u ON u.id = j.created_by` + +func scanJob(row interface{ Scan(...any) error }) (*jobView, error) { + v := &jobView{} + var result string + err := row.Scan(&v.ID, &v.Kind, &v.Title, &v.Status, &v.Total, &v.Done, &v.Message, &result, + &v.ProjectSlug, &v.ProjectName, &v.CreatedBy, &v.CreatedAt, &v.StartedAt, &v.FinishedAt, &v.UpdatedAt, &v.Cancelling) + if result != "" { + v.Result = json.RawMessage(result) + } + return v, err +} + +// visibleJobs: admins see every job; others see jobs of their projects and the ones they started. +func jobScope(u *User) (string, []any) { + if u.IsAdmin() { + return "1 = 1", nil + } + return `(j.created_by = ? OR j.project_id IN (SELECT project_id FROM project_members WHERE user_id = ?))`, []any{u.ID, u.ID} +} + +func (s *Server) listJobs(w http.ResponseWriter, r *http.Request, u *User) error { + where, args := jobScope(u) + q := r.URL.Query() + if slug := q.Get("project"); slug != "" { + where += " AND p.slug = ?" + args = append(args, slug) + } + if q.Get("active") == "1" { + where += " AND j.status IN ('queued', 'running')" + } + limit := 100 + if n, err := strconv.Atoi(q.Get("limit")); err == nil && n > 0 && n <= 500 { + limit = n + } + rows, err := s.db.QueryContext(r.Context(), jobSelect+` WHERE `+where+` ORDER BY j.id DESC LIMIT ?`, append(args, limit)...) + if err != nil { + return err + } + defer rows.Close() + list := []*jobView{} + for rows.Next() { + v, err := scanJob(rows) + if err != nil { + return err + } + list = append(list, v) + } + writeJSON(w, 200, map[string]any{"jobs": list}) + return rows.Err() +} + +func (s *Server) visibleJob(r *http.Request, u *User) (*jobView, error) { + id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) + if err != nil { + return nil, errNotFound + } + where, args := jobScope(u) + v, err := scanJob(s.db.QueryRowContext(r.Context(), jobSelect+` WHERE j.id = ? AND `+where, append([]any{id}, args...)...)) + if errors.Is(err, sql.ErrNoRows) { + return nil, errNotFound + } + return v, err +} + +func (s *Server) getJob(w http.ResponseWriter, r *http.Request, u *User) error { + v, err := s.visibleJob(r, u) + if err != nil { + return err + } + writeJSON(w, 200, map[string]any{"job": v}) + return nil +} + +func (s *Server) cancelJob(w http.ResponseWriter, r *http.Request, u *User) error { + v, err := s.visibleJob(r, u) + if err != nil { + return err + } + if !u.IsAdmin() && v.CreatedBy != u.Username { + var role string + s.db.QueryRow(`SELECT m.role FROM project_members m JOIN projects p ON p.id = m.project_id WHERE p.slug = ? AND m.user_id = ?`, v.ProjectSlug, u.ID).Scan(&role) + if roleNames[role] < roleDeveloper { + return errForbidden + } + } + switch v.Status { + case "queued": + s.finishJob(v.ID, "cancelled", "", nil) + case "running": + s.db.Exec(`UPDATE jobs SET cancel_requested = 1, updated_at = ? WHERE id = ?`, now(), v.ID) + default: + return badRequest("This job has already finished") + } + writeJSON(w, 200, map[string]bool{"ok": true}) + return nil +} + +// retryJob puts a failed or cancelled job back in the queue; it continues from its saved state. +func (s *Server) retryJob(w http.ResponseWriter, r *http.Request, u *User) error { + v, err := s.visibleJob(r, u) + if err != nil { + return err + } + if !u.IsAdmin() && v.CreatedBy != u.Username { + return errForbidden + } + if v.Status != "failed" && v.Status != "cancelled" { + return badRequest("Only failed or cancelled jobs can be retried") + } + if _, err := s.db.Exec(`UPDATE jobs SET status = 'queued', cancel_requested = 0, message = '', finished_at = NULL, updated_at = ? WHERE id = ?`, now(), v.ID); err != nil { + return err + } + s.wakeJobs() + writeJSON(w, 200, map[string]bool{"ok": true}) + return nil +} diff --git a/internal/server/load_test.go b/internal/server/load_test.go new file mode 100644 index 0000000..6cff0e2 --- /dev/null +++ b/internal/server/load_test.go @@ -0,0 +1,70 @@ +package server + +import ( + "os" + "runtime" + "strconv" + "testing" + "time" +) + +// TestTolgeeImportLoad imports a Tolgee project of LOWLEX_LOAD keys (e.g. 160000) and checks +// that the API keeps answering while it runs. Skipped unless LOWLEX_LOAD is set. +func TestTolgeeImportLoad(t *testing.T) { + n, _ := strconv.Atoi(os.Getenv("LOWLEX_LOAD")) + if n == 0 { + t.Skip("set LOWLEX_LOAD=160000 to run") + } + tg, _ := fakeTolgee(t, n, 0) + _, newClient := newTestServer(t) + admin := newClient() + admin.do("POST", "/api/auth/login", map[string]string{"username": "admin", "password": DefaultAdminPassword}, 200) + admin.do("POST", "/api/auth/password", map[string]string{"current_password": DefaultAdminPassword, "new_password": "load-test-password"}, 200) + + start := time.Now() + res := admin.do("POST", "/api/tolgee/import", map[string]any{"url": tg.URL, "api_key": "tgpak_test", "target": "new"}, 201) + id := int(res["job_id"].(float64)) + + // while it runs, the editor's key list must stay fast + var worst time.Duration + for { + t0 := time.Now() + admin.do("GET", "/api/projects/big-app/keys?per=50&page=10", nil, 200) + if d := time.Since(t0); d > worst { + worst = d + } + job := admin.do("GET", "/api/jobs/"+strconv.Itoa(id), nil, 200)["job"].(map[string]any) + if job["status"] != "running" && job["status"] != "queued" { + if job["status"] != "done" { + t.Fatalf("job: %v", job) + } + break + } + if time.Since(start) > 15*time.Minute { + t.Fatal("import too slow") + } + time.Sleep(500 * time.Millisecond) + } + took := time.Since(start) + + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + t.Logf("%d keys imported in %s (%.0f keys/s); slowest key list during import %s; heap in use %d MB", + n, took.Round(time.Millisecond), float64(n)/took.Seconds(), worst.Round(time.Millisecond), mem.HeapInuse>>20) + + // a page of the editor, a search and the project stats on the full project + for _, path := range []string{"/api/projects/big-app/keys?per=50&page=1500", "/api/projects/big-app/keys?q=Texto+15999", "/api/projects/big-app", "/api/projects"} { + t0 := time.Now() + admin.do("GET", path, nil, 200) + t.Logf("GET %s: %s", path, time.Since(t0).Round(time.Millisecond)) + } + for _, path := range []string{"/api/projects/big-app/export?format=json-nested", "/api/projects/big-app/export?format=csv"} { + t0 := time.Now() + code, body := admin.raw(path) + if code != 200 { + t.Fatalf("%s: %d", path, code) + } + runtime.ReadMemStats(&mem) + t.Logf("GET %s: %s, %d MB, heap in use %d MB", path, time.Since(t0).Round(time.Millisecond), len(body)>>20, mem.HeapInuse>>20) + } +} diff --git a/internal/server/providers.go b/internal/server/providers.go index 6d147d0..983a187 100644 --- a/internal/server/providers.go +++ b/internal/server/providers.go @@ -3,6 +3,8 @@ package server import ( "context" "encoding/json" + "errors" + "fmt" "net/http" "strconv" "strings" @@ -371,27 +373,20 @@ func (s *Server) suggest(w http.ResponseWriter, r *http.Request, pc *projectCtx) return nil } -// batchTranslate fills a language with machine translations, by default only missing cells. +// batchTranslate queues a job that fills a language with machine translations, +// by default only the missing cells. func (s *Server) batchTranslate(w http.ResponseWriter, r *http.Request, pc *projectCtx) error { - var in struct { - Lang string `json:"lang"` - ProviderID int64 `json:"provider_id"` - KeyIDs []int64 `json:"key_ids"` - Overwrite bool `json:"overwrite"` - State string `json:"state"` - Source string `json:"source"` - } + var in mtBatchParams if err := readJSON(r, &in); err != nil { return err } if !pc.canEditLang(in.Lang) { return errf(http.StatusForbidden, "language_forbidden", "You cannot edit %s in this project", in.Lang) } - source := in.Source - if source == "" { - source = pc.BaseLanguage + if in.Source == "" { + in.Source = pc.BaseLanguage } - if source == in.Lang { + if in.Source == in.Lang { return badRequest("Choose a target language different from the source") } if in.State == "" { @@ -410,91 +405,171 @@ func (s *Server) batchTranslate(w http.ResponseWriter, r *http.Request, pc *proj if !p.Enabled { return badRequest("This provider is disabled") } - srcID, err := s.languageID(r.Context(), pc.ID, source) + if _, err := s.languageID(r.Context(), pc.ID, in.Source); err != nil { + return err + } + if _, err := s.languageID(r.Context(), pc.ID, in.Lang); err != nil { + return err + } + total, err := s.countMTBatch(r.Context(), pc.ID, in, 0) if err != nil { return err } - dstID, err := s.languageID(r.Context(), pc.ID, in.Lang) + if total == 0 { + writeJSON(w, 200, map[string]any{"job_id": nil, "total": 0}) + return nil + } + in.ProjectID, in.Context = pc.ID, pc.Description + id, err := s.createJob(pc.ID, "mt.batch", p.Name+" → "+in.Lang, pc.userID, in) if err != nil { return err } + s.db.Exec(`UPDATE jobs SET total = ? WHERE id = ?`, total, id) + writeJSON(w, 201, map[string]any{"job_id": id, "total": total}) + return nil +} + +type mtBatchParams struct { + Lang string `json:"lang"` + ProviderID int64 `json:"provider_id"` + KeyIDs []int64 `json:"key_ids"` + Overwrite bool `json:"overwrite"` + State string `json:"state"` + Source string `json:"source"` + ProjectID int64 `json:"project_id"` + Context string `json:"context"` +} + +func init() { jobHandlers["mt.batch"] = runMTBatch } - q := `SELECT k.id, k.name, k.description, st.text, COALESCE(dt.state, '') - FROM keys k JOIN translations st ON st.key_id = k.id AND st.language_id = ? - LEFT JOIN translations dt ON dt.key_id = k.id AND dt.language_id = ? - WHERE k.project_id = ? AND st.text <> ''` - args := []any{srcID, dstID, pc.ID} - if in.Overwrite { +// mtBatchWhere selects the keys still to translate after key id `after`. +func mtBatchWhere(p mtBatchParams, after int64) (string, []any) { + q := `FROM keys k + JOIN languages sl ON sl.project_id = k.project_id AND sl.code = ? + JOIN languages dl ON dl.project_id = k.project_id AND dl.code = ? + JOIN translations st ON st.key_id = k.id AND st.language_id = sl.id + LEFT JOIN translations dt ON dt.key_id = k.id AND dt.language_id = dl.id + WHERE k.project_id = ? AND k.id > ? AND st.text <> ''` + args := []any{p.Source, p.Lang, p.ProjectID, after} + if p.Overwrite { q += ` AND COALESCE(dt.state, '') <> 'reviewed'` } else { q += ` AND dt.key_id IS NULL` } - if len(in.KeyIDs) > 0 { - q += ` AND k.id IN (` + strings.TrimSuffix(strings.Repeat("?,", len(in.KeyIDs)), ",") + `)` - for _, id := range in.KeyIDs { + if len(p.KeyIDs) > 0 { + q += ` AND k.id IN (` + strings.TrimSuffix(strings.Repeat("?,", len(p.KeyIDs)), ",") + `)` + for _, id := range p.KeyIDs { args = append(args, id) } } - rows, err := s.db.QueryContext(r.Context(), q+` ORDER BY k.id`, args...) - if err != nil { - return err + return q, args +} + +func (s *Server) countMTBatch(ctx context.Context, projectID int64, p mtBatchParams, after int64) (int, error) { + p.ProjectID = projectID + where, args := mtBatchWhere(p, after) + var n int + err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) `+where, args...).Scan(&n) + return n, err +} + +type mtBatchState struct { + After int64 `json:"after"` + Done int `json:"done"` + Translated int `json:"translated"` +} + +func runMTBatch(ctx context.Context, j *jobRun) (any, error) { + s := j.s + var p mtBatchParams + if err := json.Unmarshal(j.Params, &p); err != nil { + return nil, err } - var ids []int64 - var items []mt.Item - for rows.Next() { - var id int64 - var it mt.Item - var state string - if err := rows.Scan(&id, &it.Key, &it.Description, &it.Text, &state); err != nil { - rows.Close() - return err - } - ids = append(ids, id) - items = append(items, it) + var st mtBatchState + if j.State != "" { + json.Unmarshal([]byte(j.State), &st) } - rows.Close() - if len(items) == 0 { - writeJSON(w, 200, map[string]any{"translated": 0, "failed": 0}) - return nil + prov, err := s.loadProvider(ctx, p.ProviderID) + if err != nil { + return nil, fmt.Errorf("provider: %w", err) } - - t, err := mt.New(p.Type, p.Config) + t, err := mt.New(prov.Type, prov.Config) if err != nil { - return err + return nil, err } - ctx, cancel := context.WithTimeout(r.Context(), 30*time.Minute) - defer cancel() - results, terr := mt.Translate(ctx, t, mt.Request{Source: source, Target: in.Lang, Items: items, Context: pc.Description}) - - origin := "mt:" + p.Type - if p.AI { - origin = "ai:" + p.Config.Model + var dstID int64 + if err := s.db.QueryRowContext(ctx, `SELECT id FROM languages WHERE project_id = ? AND code = ?`, p.ProjectID, p.Lang).Scan(&dstID); err != nil { + return nil, fmt.Errorf("language %s is gone", p.Lang) } - ts := now() - tx, err := s.db.BeginTx(context.WithoutCancel(r.Context()), nil) - if err != nil { - return err + origin := "mt:" + prov.Type + if prov.AI { + origin = "ai:" + prov.Config.Model } - defer tx.Rollback() - for i, text := range results { - if text == "" { - continue + chunk := 50 + if prov.AI { + chunk = 25 + } + result := func() any { return map[string]int{"translated": st.Translated, "processed": st.Done} } + + for { + where, args := mtBatchWhere(p, st.After) + rows, err := s.db.QueryContext(ctx, `SELECT k.id, k.name, k.description, st.text `+where+` ORDER BY k.id LIMIT ?`, append(args, chunk)...) + if err != nil { + return result(), err } - if _, err := tx.Exec(`INSERT INTO translations (key_id, language_id, text, state, origin, updated_by, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (key_id, language_id) DO UPDATE SET text = excluded.text, state = excluded.state, origin = excluded.origin, - updated_by = excluded.updated_by, updated_at = excluded.updated_at WHERE translations.state <> 'reviewed'`, - ids[i], dstID, text, in.State, origin, nullID(pc.userID), ts); err != nil { - return err + var ids []int64 + var items []mt.Item + for rows.Next() { + var id int64 + var it mt.Item + if err := rows.Scan(&id, &it.Key, &it.Description, &it.Text); err != nil { + rows.Close() + return result(), err + } + ids = append(ids, id) + items = append(items, it) + } + rows.Close() + if len(items) == 0 { + break + } + texts, err := mt.Translate(ctx, t, mt.Request{Source: p.Source, Target: p.Lang, Items: items, Context: p.Context}) + if err != nil { + return result(), err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return result(), err + } + ts := now() + for i, text := range texts { + if text == "" { + continue + } + if _, err := tx.ExecContext(ctx, `INSERT INTO translations (key_id, language_id, text, state, origin, updated_by, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (key_id, language_id) DO UPDATE SET text = excluded.text, state = excluded.state, origin = excluded.origin, + updated_by = excluded.updated_by, updated_at = excluded.updated_at WHERE translations.state <> 'reviewed'`, + ids[i], dstID, text, p.State, origin, nullID(j.UserID), ts); err != nil { + tx.Rollback() + return result(), err + } + st.Translated++ + } + st.After = ids[len(ids)-1] + st.Done += len(ids) + b, _ := json.Marshal(st) + cerr := j.Checkpoint(tx, st.Done, string(b)) + if cerr != nil && !errors.Is(cerr, errJobCancelled) { + tx.Rollback() + return result(), cerr + } + if err := tx.Commit(); err != nil { + return result(), err + } + if cerr != nil { + return result(), cerr } } - if err := tx.Commit(); err != nil { - return err - } - s.logActivity(pc.ID, pc.userID, "mt.batch", strconv.Itoa(len(results))+" keys → "+in.Lang+" ("+p.Name+")") - resp := map[string]any{"translated": len(results), "failed": len(items) - len(results)} - if terr != nil { - resp["error"] = terr.Error() - } - writeJSON(w, 200, resp) - return nil + s.logActivity(p.ProjectID, j.UserID, "mt.batch", fmt.Sprintf("%d keys → %s (%s)", st.Translated, p.Lang, prov.Name)) + return result(), nil } diff --git a/internal/server/server.go b/internal/server/server.go index 0c3eb20..2ca7ab2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -32,11 +32,13 @@ type Server struct { ui fs.FS mux *http.ServeMux limiter *loginLimiter + jobs *jobRunner } func New(db *sql.DB, cfg config.Config, box *secure.Box, ui fs.FS) *Server { s := &Server{db: db, cfg: cfg, box: box, ui: ui, mux: http.NewServeMux(), limiter: newLoginLimiter()} s.routes() + s.startJobs() return s } @@ -165,6 +167,14 @@ func (s *Server) routes() { m.Handle("POST /api/providers/{id}/test", s.user(s.testProvider, adminOnly)) // projects + m.Handle("GET /api/jobs", s.user(s.listJobs, 0)) + m.Handle("GET /api/jobs/{id}", s.user(s.getJob, 0)) + m.Handle("POST /api/jobs/{id}/cancel", s.user(s.cancelJob, 0)) + m.Handle("POST /api/jobs/{id}/retry", s.user(s.retryJob, 0)) + + m.Handle("POST /api/tolgee/inspect", s.user(s.tolgeeInspect, adminOnly)) + m.Handle("POST /api/tolgee/import", s.user(s.tolgeeImport, adminOnly)) + m.Handle("GET /api/projects", s.user(s.listProjects, 0)) m.Handle("POST /api/projects", s.user(s.createProject, adminOnly)) m.Handle("GET /api/projects/{slug}", s.project(roleViewer, s.getProject)) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 5449bc3..8fcc236 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/LowBloat/LowLex/internal/config" "github.com/LowBloat/LowLex/internal/secure" @@ -39,8 +40,10 @@ func newTestServer(t *testing.T) (*httptest.Server, func() *client) { if err != nil { t.Fatal(err) } - ts := httptest.NewServer(New(db, config.Config{}, box, nil)) + app := New(db, config.Config{}, box, nil) + ts := httptest.NewServer(app) t.Cleanup(ts.Close) + t.Cleanup(app.Close) return ts, func() *client { jar, _ := cookiejar.New(nil) return &client{t: t, base: ts.URL, http: &http.Client{Jar: jar}} @@ -74,6 +77,19 @@ func (c *client) do(method, path string, body any, want int) map[string]any { return out } +// waitJob polls a background job until it leaves the queue or a 10 s timeout. +func (c *client) waitJob(id int) map[string]any { + c.t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + job := c.do("GET", fmt.Sprintf("/api/jobs/%d", id), nil, 200)["job"].(map[string]any) + if st := job["status"]; (st != "queued" && st != "running") || time.Now().After(deadline) { + return job + } + time.Sleep(20 * time.Millisecond) + } +} + func (c *client) raw(path string) (int, string) { c.t.Helper() req, _ := http.NewRequest("GET", c.base+path, nil) @@ -237,10 +253,13 @@ func TestAIProvider(t *testing.T) { admin.do("POST", "/api/projects", map[string]any{"name": "App", "languages": []string{"pt-BR"}}, 201) admin.upload("/api/projects/app/import", "en.json", `{"a":"One","b":"Two","c":"Three"}`, nil, 200) - res := admin.do("POST", "/api/projects/app/mt/batch", map[string]any{"lang": "pt-BR", "provider_id": pid}, 200) - if res["translated"].(float64) != 3 { + res := admin.do("POST", "/api/projects/app/mt/batch", map[string]any{"lang": "pt-BR", "provider_id": pid}, 201) + if res["total"].(float64) != 3 { t.Fatalf("batch: %v", res) } + if job := admin.waitJob(int(res["job_id"].(float64))); job["status"] != "done" || job["done"].(float64) != 3 { + t.Fatalf("batch job: %v", job) + } keys := admin.do("GET", "/api/projects/app/keys?q=One", nil, 200)["keys"].([]any) tr := keys[0].(map[string]any)["translations"].(map[string]any)["pt-BR"].(map[string]any) if tr["text"] != "PT:One" || tr["state"] != "draft" || tr["origin"] != "ai:test" { diff --git a/internal/server/tolgee_import.go b/internal/server/tolgee_import.go new file mode 100644 index 0000000..48a6d90 --- /dev/null +++ b/internal/server/tolgee_import.go @@ -0,0 +1,333 @@ +package server + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/LowBloat/LowLex/internal/tolgee" +) + +// Importing straight from a Tolgee server: the admin gives the Tolgee URL and an API key, +// LowLex reads every key page by page in a background job and can resume after a restart. + +const tolgeePageSize = 500 + +func init() { jobHandlers["tolgee.import"] = runTolgeeImport } + +type tolgeeSource struct { + URL string `json:"url"` + APIKey string `json:"api_key"` + ProjectID int64 `json:"project_id"` +} + +func (t tolgeeSource) client() (*tolgee.Client, error) { + u := strings.TrimRight(strings.TrimSpace(t.URL), "/") + if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") { + return nil, badRequest("The Tolgee URL must start with http:// or https://") + } + // people paste the app URL; the API lives at the same origin + if i := strings.Index(u, "/projects"); i > 0 { + u = u[:i] + } + if strings.TrimSpace(t.APIKey) == "" { + return nil, badRequest("Enter a Tolgee API key") + } + return &tolgee.Client{BaseURL: u, APIKey: strings.TrimSpace(t.APIKey), ProjectID: t.ProjectID}, nil +} + +func (s *Server) tolgeeInspect(w http.ResponseWriter, r *http.Request, u *User) error { + var in tolgeeSource + if err := readJSON(r, &in); err != nil { + return err + } + c, err := in.client() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second) + defer cancel() + info, err := c.Inspect(ctx) + if err != nil { + return errf(http.StatusBadGateway, "tolgee", "%v", err) + } + writeJSON(w, 200, map[string]any{"tolgee": info}) + return nil +} + +type tolgeeImportParams struct { + tolgeeSource + Overwrite bool `json:"overwrite"` + Languages []string `json:"languages"` +} + +func (s *Server) tolgeeImport(w http.ResponseWriter, r *http.Request, u *User) error { + var in struct { + tolgeeSource + Target string `json:"target"` // "new" or "existing" + ProjectSlug string `json:"project_slug"` + Name string `json:"name"` + Slug string `json:"slug"` + Overwrite bool `json:"overwrite"` + } + if err := readJSON(r, &in); err != nil { + return err + } + c, err := in.client() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second) + defer cancel() + info, err := c.Inspect(ctx) + if err != nil { + return errf(http.StatusBadGateway, "tolgee", "%v", err) + } + in.ProjectID = info.ProjectID + + var proj *Project + if in.Target == "existing" { + if proj, err = s.loadProject(r.Context(), in.ProjectSlug); err != nil { + return err + } + } else { + name := strings.TrimSpace(in.Name) + if name == "" { + name = info.ProjectName + } + slug := in.Slug + if slug == "" { + slug = slugify(name) + } + if !slugRe.MatchString(slug) { + return badRequest("Slug: lowercase letters, digits and dashes") + } + var exists int + s.db.QueryRow(`SELECT COUNT(*) FROM projects WHERE slug = ?`, slug).Scan(&exists) + if exists > 0 { + return errf(http.StatusConflict, "slug_taken", "A project with this slug already exists") + } + base := info.BaseTag + if base == "" && len(info.Languages) > 0 { + base = info.Languages[0].Tag + } + if base == "" { + base = "en" + } + res, err := s.db.Exec(`INSERT INTO projects (slug, name, description, base_language) VALUES (?, ?, ?, ?)`, slug, name, info.Description, base) + if err != nil { + return err + } + id, _ := res.LastInsertId() + proj = &Project{ID: id, Slug: slug, Name: name, BaseLanguage: base} + s.logActivity(id, u.ID, "project.create", name+" (from Tolgee)") + } + + tags := make([]string, len(info.Languages)) + for i, l := range info.Languages { + tags[i] = l.Tag + } + params := tolgeeImportParams{tolgeeSource: in.tolgeeSource, Overwrite: in.Overwrite, Languages: tags} + jobID, err := s.createJob(proj.ID, "tolgee.import", "Tolgee: "+info.ProjectName, u.ID, params) + if err != nil { + return err + } + s.db.Exec(`UPDATE jobs SET total = ? WHERE id = ?`, info.Keys, jobID) + writeJSON(w, 201, map[string]any{"job_id": jobID, "project_slug": proj.Slug}) + return nil +} + +type tolgeeImportState struct { + Cursor string `json:"cursor,omitempty"` + Page int `json:"page,omitempty"` + Keys int `json:"keys"` + Translations int `json:"translations"` + Finished bool `json:"finished,omitempty"` +} + +type tolgeeImportResult struct { + Keys int `json:"keys"` + Translations int `json:"translations"` + LanguagesAdded []string `json:"languages_added"` +} + +func runTolgeeImport(ctx context.Context, j *jobRun) (any, error) { + s := j.s + var p tolgeeImportParams + if err := json.Unmarshal(j.Params, &p); err != nil { + return nil, err + } + var st tolgeeImportState + if j.State != "" { + json.Unmarshal([]byte(j.State), &st) + } + c, err := p.client() + if err != nil { + return nil, err + } + result := &tolgeeImportResult{LanguagesAdded: []string{}} + + // languages first, so every translation has somewhere to go + langs, err := c.Languages(ctx) + if err != nil { + return nil, err + } + langIDs := map[string]int64{} + rows, err := s.db.QueryContext(ctx, `SELECT id, code FROM languages WHERE project_id = ?`, j.ProjectID) + if err != nil { + return nil, err + } + for rows.Next() { + var id int64 + var code string + rows.Scan(&id, &code) + langIDs[code] = id + } + rows.Close() + tags := []string{} + for _, l := range langs { + tags = append(tags, l.Tag) + if _, ok := langIDs[l.Tag]; ok { + continue + } + name := l.Name + if name == "" { + name = languageName(l.Tag) + } + res, err := s.db.Exec(`INSERT INTO languages (project_id, code, name) VALUES (?, ?, ?)`, j.ProjectID, l.Tag, name) + if err != nil { + return nil, err + } + langIDs[l.Tag], _ = res.LastInsertId() + result.LanguagesAdded = append(result.LanguagesAdded, l.Tag) + } + + for !st.Finished { + if j.Cancelled() { + return result, errJobCancelled + } + page, err := c.Page(ctx, tags, st.Cursor, tolgeePageSize, st.Page) + if err != nil { + return result, err + } + // with a cursor, Tolgee counts the keys left from that point, not the whole project + if page.Total > 0 { + total := page.Total + if st.Cursor != "" { + total += st.Keys + } + if total != j.Total { + j.SetTotal(total) + } + } + // the page and the resume point are saved together + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return result, err + } + added, err := storeTolgeePage(ctx, tx, j.ProjectID, langIDs, page.Keys, p.Overwrite) + if err != nil { + tx.Rollback() + return result, err + } + st.Keys += len(page.Keys) + st.Translations += added + switch { + case len(page.Keys) == 0: + st.Finished = true + case page.NextCursor != "": + st.Cursor = page.NextCursor + case page.Pages > 0 && page.Number+1 < page.Pages: + st.Cursor, st.Page = "", page.Number+1 + default: + st.Finished = true + } + b, _ := json.Marshal(st) + cerr := j.Checkpoint(tx, st.Keys, string(b)) + if cerr != nil && !errors.Is(cerr, errJobCancelled) { + tx.Rollback() + return result, cerr + } + if err := tx.Commit(); err != nil { + return result, err + } + if cerr != nil { + result.Keys, result.Translations = st.Keys, st.Translations + return result, cerr + } + } + result.Keys, result.Translations = st.Keys, st.Translations + s.logActivity(j.ProjectID, j.UserID, "import", fmt.Sprintf("Tolgee: %d keys, %d translations", st.Keys, st.Translations)) + return result, nil +} + +// storeTolgeePage upserts one page of keys and their translated texts. +func storeTolgeePage(ctx context.Context, tx *sql.Tx, projectID int64, langIDs map[string]int64, keys []tolgee.Key, overwrite bool) (int, error) { + if len(keys) == 0 { + return 0, nil + } + keyStmt, err := tx.PrepareContext(ctx, `INSERT INTO keys (project_id, namespace, name, description, tags) VALUES (?, ?, ?, ?, ?) + ON CONFLICT (project_id, namespace, name) DO UPDATE SET + description = CASE WHEN keys.description = '' OR ? THEN excluded.description ELSE keys.description END, + tags = CASE WHEN keys.tags = '[]' OR ? THEN excluded.tags ELSE keys.tags END, + updated_at = excluded.updated_at + RETURNING id`) + if err != nil { + return 0, err + } + defer keyStmt.Close() + trStmt, err := tx.PrepareContext(ctx, `INSERT INTO translations (key_id, language_id, text, state, origin) VALUES (?, ?, ?, ?, 'import:tolgee') + ON CONFLICT (key_id, language_id) DO UPDATE SET text = excluded.text, state = excluded.state, origin = excluded.origin, + updated_at = excluded.updated_at WHERE ?`) + if err != nil { + return 0, err + } + defer trStmt.Close() + + added := 0 + for _, k := range keys { + ns, desc := "", "" + if k.Namespace != nil { + ns = *k.Namespace + } + if k.Description != nil { + desc = *k.Description + } + tagNames := make([]string, len(k.Tags)) + for i, t := range k.Tags { + tagNames[i] = t.Name + } + var keyID int64 + if err := keyStmt.QueryRowContext(ctx, projectID, ns, k.Name, desc, cleanTags(tagNames), overwrite, overwrite).Scan(&keyID); err != nil { + return 0, fmt.Errorf("key %q: %w", k.Name, err) + } + for tag, tr := range k.Translations { + lid, ok := langIDs[tag] + if !ok || tr.Text == "" { + continue + } + state := "" + switch tr.State { + case "REVIEWED": + state = "reviewed" + case "TRANSLATED": + state = "translated" + default: + continue // UNTRANSLATED, DISABLED + } + res, err := trStmt.ExecContext(ctx, keyID, lid, tr.Text, state, overwrite) + if err != nil { + return 0, fmt.Errorf("key %q [%s]: %w", k.Name, tag, err) + } + if n, _ := res.RowsAffected(); n > 0 { + added++ + } + } + } + return added, nil +} diff --git a/internal/server/tolgee_test.go b/internal/server/tolgee_test.go new file mode 100644 index 0000000..f3cd1a6 --- /dev/null +++ b/internal/server/tolgee_test.go @@ -0,0 +1,169 @@ +package server + +import ( + "archive/zip" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" +) + +// fakeTolgee serves a Tolgee project with n keys in three languages, paged by cursor. +// When failAt > 0, the first request for that page answers 400 once. +func fakeTolgee(t *testing.T, n int, failAt int) (*httptest.Server, *atomic.Int32) { + var pagesServed atomic.Int32 + var failed atomic.Bool + mux := http.NewServeMux() + auth := func(h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-API-Key") != "tgpak_test" { + w.WriteHeader(401) + return + } + w.Header().Set("Content-Type", "application/json") + h(w, r) + } + } + mux.HandleFunc("GET /v2/api-keys/current", auth(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"projectId": 7, "projectName": "Big app"}`) + })) + mux.HandleFunc("GET /v2/projects/7", auth(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"id": 7, "name": "Big app", "description": "A big app", "baseLanguage": {"tag": "en"}}`) + })) + mux.HandleFunc("GET /v2/projects/7/languages", auth(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"_embedded": {"languages": [ + {"id": 1, "tag": "en", "name": "English", "base": true}, + {"id": 2, "tag": "pt-BR", "name": "Portuguese (Brazil)"}, + {"id": 3, "tag": "fr", "name": "French"}]}}`) + })) + mux.HandleFunc("GET /v2/projects/7/translations", auth(func(w http.ResponseWriter, r *http.Request) { + size, _ := strconv.Atoi(r.URL.Query().Get("size")) + start, _ := strconv.Atoi(r.URL.Query().Get("cursor")) + page := start / max(size, 1) + if failAt > 0 && page == failAt && !failed.Swap(true) { + w.WriteHeader(400) + fmt.Fprint(w, `{"code": "bad"}`) + return + } + if size > 1 { + pagesServed.Add(1) + } + keys := []map[string]any{} + for i := start; i < min(start+size, n); i++ { + var ns any + if i%2 == 0 { + ns = "common" + } + tr := map[string]any{ + "en": map[string]any{"text": fmt.Sprintf("Text %d", i), "state": "REVIEWED"}, + "pt-BR": map[string]any{"text": fmt.Sprintf("Texto %d", i), "state": "TRANSLATED"}, + "fr": map[string]any{"text": nil, "state": "UNTRANSLATED"}, + } + keys = append(keys, map[string]any{"keyId": i + 1, "keyName": fmt.Sprintf("key.%d", i), "keyNamespace": ns, + "keyDescription": "ctx", "keyTags": []map[string]any{{"id": 1, "name": "web"}}, "translations": tr}) + } + // like Tolgee: after a cursor, totalElements counts only the keys left + res := map[string]any{"_embedded": map[string]any{"keys": keys}, "page": map[string]any{"totalElements": n - start, "number": 0}} + if start+size < n { + res["nextCursor"] = strconv.Itoa(start + size) + } + json.NewEncoder(w).Encode(res) + })) + ts := httptest.NewServer(mux) + t.Cleanup(ts.Close) + return ts, &pagesServed +} + +func TestTolgeeImport(t *testing.T) { + const n = 1234 // three pages of 500 + tg, pages := fakeTolgee(t, n, 1) + _, newClient := newTestServer(t) + admin := newClient() + admin.do("POST", "/api/auth/login", map[string]string{"username": "admin", "password": DefaultAdminPassword}, 200) + admin.do("POST", "/api/auth/password", map[string]string{"current_password": DefaultAdminPassword, "new_password": "tolgee-test-password"}, 200) + + admin.do("POST", "/api/tolgee/inspect", map[string]any{"url": tg.URL, "api_key": "wrong"}, 502) + info := admin.do("POST", "/api/tolgee/inspect", map[string]any{"url": tg.URL + "/projects/7/translations", "api_key": "tgpak_test"}, 200)["tolgee"].(map[string]any) + if info["project_name"] != "Big app" || info["keys"].(float64) != n || len(info["languages"].([]any)) != 3 { + t.Fatalf("inspect: %v", info) + } + + res := admin.do("POST", "/api/tolgee/import", map[string]any{"url": tg.URL, "api_key": "tgpak_test", "target": "new"}, 201) + if res["project_slug"] != "big-app" { + t.Fatalf("import: %v", res) + } + id := int(res["job_id"].(float64)) + + // page 2 fails once: the job stops, keeps page 1, and a retry continues from page 2 + job := admin.waitJob(id) + if job["status"] != "failed" || job["done"].(float64) != 500 { + t.Fatalf("expected a failure after the first page: %v", job) + } + admin.do("POST", fmt.Sprintf("/api/jobs/%d/retry", id), nil, 200) + job = admin.waitJob(id) + if job["status"] != "done" || job["done"].(float64) != n || job["total"].(float64) != n { + t.Fatalf("retry: %v", job) + } + if got := pages.Load(); got != 3 { + t.Fatalf("pages fetched: %d, want 3 (resume must not refetch page 1)", got) + } + result := job["result"].(map[string]any) + if result["keys"].(float64) != n || result["translations"].(float64) != 2*n { + t.Fatalf("result: %v", result) + } + + p := admin.do("GET", "/api/projects/big-app", nil, 200) + if p["keys"].(float64) != n || p["project"].(map[string]any)["base_language"] != "en" { + t.Fatalf("project: %v", p) + } + for _, l := range p["languages"].([]any) { + l := l.(map[string]any) + want := map[string]float64{"en": n, "pt-BR": n, "fr": 0}[l["code"].(string)] + if l["translated"].(float64) != want { + t.Fatalf("language %v: translated %v, want %v", l["code"], l["translated"], want) + } + if l["code"] == "en" && l["reviewed"].(float64) != n { + t.Fatalf("reviewed state lost: %v", l) + } + } + ns := admin.do("GET", "/api/projects/big-app/namespaces", nil, 200) + if len(ns["namespaces"].([]any)) != 2 || ns["tags"].([]any)[0] != "web" { + t.Fatalf("namespaces/tags: %v", ns) + } + + // running it again changes nothing without overwrite and duplicates nothing + res = admin.do("POST", "/api/tolgee/import", map[string]any{"url": tg.URL, "api_key": "tgpak_test", "target": "existing", "project_slug": "big-app"}, 201) + job = admin.waitJob(int(res["job_id"].(float64))) + if job["status"] != "done" || job["result"].(map[string]any)["translations"].(float64) != 0 { + t.Fatalf("second import: %v", job) + } + if p := admin.do("GET", "/api/projects/big-app", nil, 200); p["keys"].(float64) != n { + t.Fatalf("duplicated keys: %v", p["keys"]) + } + + // streamed exports: CSV merges every language per key, ZIP has one file per language and namespace + code, csvBody := admin.raw("/api/projects/big-app/export?format=csv") + lines := strings.Split(strings.TrimSpace(csvBody), "\n") + if code != 200 || len(lines) != n+1 || !strings.Contains(csvBody, "key,description,en,fr,pt-BR") || !strings.Contains(csvBody, "common:key.0,ctx,Text 0,,Texto 0") { + t.Fatalf("csv export: %d, %d lines\n%s", code, len(lines), lines[:3]) + } + code, zipBody := admin.raw("/api/projects/big-app/export?format=json-nested") + zr, err := zip.NewReader(strings.NewReader(zipBody), int64(len(zipBody))) + if code != 200 || err != nil || len(zr.File) != 6 { + t.Fatalf("zip export: %d %v", code, err) + } + code, nsBody := admin.raw("/api/projects/big-app/export?format=csv&ns=common") + if code != 200 || len(strings.Split(strings.TrimSpace(nsBody), "\n")) != n/2+1 { + t.Fatalf("csv export of one namespace: %d", code) + } + + // jobs are listed for the project + jobs := admin.do("GET", "/api/jobs?project=big-app", nil, 200)["jobs"].([]any) + if len(jobs) != 2 { + t.Fatalf("jobs: %v", jobs) + } +} diff --git a/internal/store/schema2.sql b/internal/store/schema2.sql new file mode 100644 index 0000000..d930765 --- /dev/null +++ b/internal/store/schema2.sql @@ -0,0 +1,25 @@ +-- LowLex schema, version 2: background jobs. +-- params holds the job input sealed with the provider key box (it may carry API keys); +-- state is the resume point (a cursor or the last processed id), saved with each batch. + +CREATE TABLE jobs ( + id INTEGER PRIMARY KEY, + project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE, + kind TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'running', 'done', 'failed', 'cancelled')), + total INTEGER NOT NULL DEFAULT 0, + done INTEGER NOT NULL DEFAULT 0, + message TEXT NOT NULL DEFAULT '', + params TEXT NOT NULL DEFAULT '', + state TEXT NOT NULL DEFAULT '', + result TEXT NOT NULL DEFAULT '', + cancel_requested INTEGER NOT NULL DEFAULT 0, + created_by INTEGER REFERENCES users(id) ON DELETE SET NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + started_at TEXT, + finished_at TEXT, + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) +); +CREATE INDEX jobs_status ON jobs(status, id); +CREATE INDEX jobs_project ON jobs(project_id, id); diff --git a/internal/store/store.go b/internal/store/store.go index 3fc81d9..436ea6d 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -14,8 +14,11 @@ import ( //go:embed schema.sql var schemaV1 string +//go:embed schema2.sql +var schemaV2 string + // migrations[i] moves the database from user_version i to i+1. -var migrations = []string{schemaV1} +var migrations = []string{schemaV1, schemaV2} // Open creates the data directory if needed, opens lowlex.db and migrates it. func Open(dataDir string) (*sql.DB, error) { diff --git a/internal/tolgee/tolgee.go b/internal/tolgee/tolgee.go new file mode 100644 index 0000000..ceedd62 --- /dev/null +++ b/internal/tolgee/tolgee.go @@ -0,0 +1,261 @@ +// Package tolgee reads projects from a Tolgee server through its REST API (v2), so they can be +// moved into LowLex without exporting files. +// +// Both kinds of Tolgee keys work: a project API key (tgpak_…) knows its project; a personal +// access token (tgpat_…) needs the project id. +package tolgee + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +type Client struct { + BaseURL string + APIKey string + ProjectID int64 + HTTP *http.Client +} + +type Language struct { + ID int64 `json:"id"` + Tag string `json:"tag"` + Name string `json:"name"` + OriginalName string `json:"originalName"` + Base bool `json:"base"` +} + +type Info struct { + ProjectID int64 `json:"project_id"` + ProjectName string `json:"project_name"` + Description string `json:"description"` + BaseTag string `json:"base_language"` + Languages []Language `json:"languages"` + Keys int `json:"keys"` +} + +type Translation struct { + Text string `json:"text"` + State string `json:"state"` // UNTRANSLATED, TRANSLATED, REVIEWED, DISABLED +} + +type Key struct { + ID int64 `json:"keyId"` + Name string `json:"keyName"` + Namespace *string `json:"keyNamespace"` + Description *string `json:"keyDescription"` + Tags []struct{ Name string } `json:"keyTags"` + Translations map[string]Translation `json:"translations"` +} + +type Page struct { + Keys []Key + Total int + NextCursor string + Number int + Pages int +} + +func (c *Client) client() *http.Client { + if c.HTTP != nil { + return c.HTTP + } + return &http.Client{Timeout: 2 * time.Minute} +} + +// Error is a non-2xx answer from Tolgee. +type Error struct { + Status int + Body string +} + +func (e *Error) Error() string { + switch e.Status { + case 401: + return "Tolgee rejected the API key (401)" + case 403: + return "the API key cannot read this project's translations (403); it needs the translations.view scope" + case 404: + return "Tolgee answered 404: check the URL and the project id" + } + return fmt.Sprintf("Tolgee answered %d: %s", e.Status, e.Body) +} + +func (c *Client) get(ctx context.Context, path string, q url.Values, out any) error { + base := strings.TrimRight(c.BaseURL, "/") + u := base + path + if len(q) > 0 { + u += "?" + q.Encode() + } + var lastErr error + for attempt := range 6 { + if attempt > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(1<= 500 { + lastErr = &Error{resp.StatusCode, snippet(body)} + continue + } + if resp.StatusCode >= 300 { + return &Error{resp.StatusCode, snippet(body)} + } + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("unexpected answer from %s: %w", path, err) + } + return nil + } + return lastErr +} + +func snippet(b []byte) string { + s := strings.TrimSpace(string(b)) + if len(s) > 300 { + s = s[:300] + "…" + } + return s +} + +// resolveProject finds the project id of a project API key when none was given. +func (c *Client) resolveProject(ctx context.Context) error { + if c.ProjectID > 0 { + return nil + } + var cur struct { + ProjectID int64 `json:"projectId"` + } + if err := c.get(ctx, "/v2/api-keys/current", nil, &cur); err != nil { + var te *Error + if errors.As(err, &te) && (te.Status == 400 || te.Status == 403 || te.Status == 404) { + return errors.New("this key is not a project API key; enter the Tolgee project id") + } + return err + } + if cur.ProjectID == 0 { + return errors.New("enter the Tolgee project id") + } + c.ProjectID = cur.ProjectID + return nil +} + +func (c *Client) project() string { return "/v2/projects/" + strconv.FormatInt(c.ProjectID, 10) } + +// Inspect returns the project's name, languages and key count. +func (c *Client) Inspect(ctx context.Context) (*Info, error) { + if err := c.resolveProject(ctx); err != nil { + return nil, err + } + var p struct { + ID int64 `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + BaseLanguage *struct { + Tag string `json:"tag"` + } `json:"baseLanguage"` + } + if err := c.get(ctx, c.project(), nil, &p); err != nil { + return nil, err + } + info := &Info{ProjectID: c.ProjectID, ProjectName: p.Name, Description: p.Description} + if p.BaseLanguage != nil { + info.BaseTag = p.BaseLanguage.Tag + } + langs, err := c.Languages(ctx) + if err != nil { + return nil, err + } + info.Languages = langs + for _, l := range langs { + if l.Base && info.BaseTag == "" { + info.BaseTag = l.Tag + } + } + page, err := c.Page(ctx, nil, "", 1, 0) + if err != nil { + return nil, err + } + info.Keys = page.Total + return info, nil +} + +func (c *Client) Languages(ctx context.Context) ([]Language, error) { + var res struct { + Embedded struct { + Languages []Language `json:"languages"` + } `json:"_embedded"` + } + if err := c.get(ctx, c.project()+"/languages", url.Values{"size": {"1000"}}, &res); err != nil { + return nil, err + } + return res.Embedded.Languages, nil +} + +// Page reads one page of keys with their translations. Tolgee pages with an opaque cursor; +// pageNum is the fallback for servers that do not return one. +func (c *Client) Page(ctx context.Context, langs []string, cursor string, size, pageNum int) (*Page, error) { + if err := c.resolveProject(ctx); err != nil { + return nil, err + } + q := url.Values{"size": {strconv.Itoa(size)}, "sort": {"keyId"}} + for _, l := range langs { + q.Add("languages", l) + } + if cursor != "" { + q.Set("cursor", cursor) + } else if pageNum > 0 { + q.Set("page", strconv.Itoa(pageNum)) + } + var res struct { + Embedded struct { + Keys []Key `json:"keys"` + } `json:"_embedded"` + Page struct { + TotalElements int `json:"totalElements"` + TotalPages int `json:"totalPages"` + Number int `json:"number"` + } `json:"page"` + NextCursor string `json:"nextCursor"` + } + if err := c.get(ctx, c.project()+"/translations", q, &res); err != nil { + return nil, err + } + return &Page{ + Keys: res.Embedded.Keys, + Total: res.Page.TotalElements, + NextCursor: res.NextCursor, + Number: res.Page.Number, + Pages: res.Page.TotalPages, + }, nil +} diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 6495226..f850cfc 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -197,3 +197,33 @@ export interface Meta { languages: Record; formats: { id: string; label: string; ext: string }[]; } + +export type JobStatus = 'queued' | 'running' | 'done' | 'failed' | 'cancelled'; + +export interface Job { + id: number; + kind: 'tolgee.import' | 'mt.batch' | string; + title: string; + status: JobStatus; + total: number; + done: number; + message: string; + result?: Record; + project_slug?: string; + project_name?: string; + created_by: string; + created_at: string; + started_at: string | null; + finished_at: string | null; + updated_at: string; + cancelling: boolean; +} + +export interface TolgeeInfo { + project_id: number; + project_name: string; + description: string; + base_language: string; + languages: { id: number; tag: string; name: string; base: boolean }[]; + keys: number; +} diff --git a/web/src/lib/components/BatchTranslateDialog.svelte b/web/src/lib/components/BatchTranslateDialog.svelte index fc49ff8..860caab 100644 --- a/web/src/lib/components/BatchTranslateDialog.svelte +++ b/web/src/lib/components/BatchTranslateDialog.svelte @@ -1,7 +1,8 @@ @@ -81,11 +113,15 @@ {#if project.can.review} {/if} - {#if busy}{/if} + {#if job} + + {num(job.done)} / {num(job.total)} · {percent(job)}% +

{t('batch.background')}

+ {/if} {/if} {#snippet actions()} - + {/snippet} diff --git a/web/src/lib/components/TolgeeImportDialog.svelte b/web/src/lib/components/TolgeeImportDialog.svelte new file mode 100644 index 0000000..557d605 --- /dev/null +++ b/web/src/lib/components/TolgeeImportDialog.svelte @@ -0,0 +1,150 @@ + + + + + + {#if error}{error}{/if} + + + + + + + + {#if info} + + + {t('tolgee.found', { keys: num(info.keys), langs: info.languages.length })} + + {#each info.languages as l (l.tag)} + + {/each} + + + + + {#if target === 'new'} + + + {:else} +