Wait out Tolgee's rate limit instead of failing the import

internal/server/tolgee_import.go

@@ -170,6 +170,8 @@ func runTolgeeImport(ctx context.Context, j *jobRun) (any, error) {
 	if err != nil {
 		return nil, err
 	}
+	// shown on the Tasks page as "waiting for Tolgee's rate limit"
+	c.OnRateLimit = func(wait time.Duration) { j.Note(fmt.Sprintf("rate_limited:%d", int(wait.Seconds()))) }
 	result := &tolgeeImportResult{LanguagesAdded: []string{}}
 
 	// languages first, so every translation has somewhere to go

internal/server/tolgee_test.go

@@ -13,10 +13,11 @@ import (
 )
 
 // fakeTolgee serves a Tolgee project with n keys in three languages, paged by cursor.
-// When failAt > 0, the first request for that page answers 400 once.
-func fakeTolgee(t *testing.T, n int, failAt int) (*httptest.Server, *atomic.Int32) {
+// When failAt > 0, the first request for that page answers 400 once; when limitAt > 0, the
+// first request for that page answers 429 with a short retryAfter, like Tolgee's rate limiter.
+func fakeTolgee(t *testing.T, n int, failAt int, limitAt ...int) (*httptest.Server, *atomic.Int32) {
 	var pagesServed atomic.Int32
-	var failed atomic.Bool
+	var failed, limited atomic.Bool
 	mux := http.NewServeMux()
 	auth := func(h http.HandlerFunc) http.HandlerFunc {
 		return func(w http.ResponseWriter, r *http.Request) {
@@ -44,6 +45,11 @@ func fakeTolgee(t *testing.T, n int, failAt int) (*httptest.Server, *atomic.Int3
 		size, _ := strconv.Atoi(r.URL.Query().Get("size"))
 		start, _ := strconv.Atoi(r.URL.Query().Get("cursor"))
 		page := start / max(size, 1)
+		if len(limitAt) > 0 && page == limitAt[0] && !limited.Swap(true) {
+			w.WriteHeader(429)
+			fmt.Fprint(w, `{"message": "rate_limited", "retryAfter": 20, "global": false}`)
+			return
+		}
 		if failAt > 0 && page == failAt && !failed.Swap(true) {
 			w.WriteHeader(400)
 			fmt.Fprint(w, `{"code": "bad"}`)
@@ -167,3 +173,16 @@ func TestTolgeeImport(t *testing.T) {
 		t.Fatalf("jobs: %v", jobs)
 	}
 }
+
+func TestTolgeeImportWaitsOutRateLimit(t *testing.T) {
+	tg, pages := fakeTolgee(t, 1200, 0, 1)
+	_, newClient := newTestServer(t)
+	admin := newClient()
+	admin.do("POST", "/api/auth/login", map[string]string{"username": "admin", "password": DefaultAdminPassword}, 200)
+	admin.do("POST", "/api/auth/password", map[string]string{"current_password": DefaultAdminPassword, "new_password": "rate-limit-password"}, 200)
+	res := admin.do("POST", "/api/tolgee/import", map[string]any{"url": tg.URL, "api_key": "tgpak_test", "target": "new"}, 201)
+	job := admin.waitJob(int(res["job_id"].(float64)))
+	if job["status"] != "done" || job["done"].(float64) != 1200 || pages.Load() != 3 {
+		t.Fatalf("rate-limited import: %v (pages %d)", job, pages.Load())
+	}
+}

internal/tolgee/tolgee.go

@@ -23,6 +23,8 @@ type Client struct {
 	APIKey    string
 	ProjectID int64
 	HTTP      *http.Client
+	// OnRateLimit is told how long the client waits after a 429.
+	OnRateLimit func(wait time.Duration)
 }
 
 type Language struct {
@@ -89,55 +91,87 @@ func (e *Error) Error() string {
 	return fmt.Sprintf("Tolgee answered %d: %s", e.Status, e.Body)
 }
 
+// MaxRateLimitWait bounds how long one request keeps waiting out Tolgee's rate limits.
+// A self-hosted Tolgee allows 100 translation requests per user every 5 minutes by default.
+const MaxRateLimitWait = 2 * time.Hour
+
 func (c *Client) get(ctx context.Context, path string, q url.Values, out any) error {
 	base := strings.TrimRight(c.BaseURL, "/")
 	u := base + path
 	if len(q) > 0 {
 		u += "?" + q.Encode()
 	}
 	var lastErr error
-	for attempt := range 6 {
-		if attempt > 0 {
-			select {
-			case <-ctx.Done():
-				return ctx.Err()
-			case <-time.After(time.Duration(1<<attempt) * time.Second):
-			}
-		}
+	failures := 0
+	var waited time.Duration
+	for {
 		req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
 		if err != nil {
 			return err
 		}
 		req.Header.Set("X-API-Key", c.APIKey)
 		req.Header.Set("Accept", "application/json")
 		resp, err := c.client().Do(req)
-		if err != nil {
+		var body []byte
+		if err == nil {
+			body, err = io.ReadAll(io.LimitReader(resp.Body, 256<<20))
+			resp.Body.Close()
+		}
+		var wait time.Duration
+		switch {
+		case err != nil:
 			if ctx.Err() != nil {
 				return ctx.Err()
 			}
 			lastErr = err
-			continue
-		}
-		body, err := io.ReadAll(io.LimitReader(resp.Body, 256<<20))
-		resp.Body.Close()
-		if err != nil {
-			lastErr = err
-			continue
-		}
-		// rate limits and server hiccups are retried; anything else is final
-		if resp.StatusCode == 429 || resp.StatusCode >= 500 {
+		case resp.StatusCode == 429:
+			// wait as long as Tolgee asks; hammering it only adds strikes and gets the connection dropped
+			wait = retryAfter(resp, body)
+			if waited+wait > MaxRateLimitWait {
+				return &Error{429, snippet(body)}
+			}
+			waited += wait
+			if c.OnRateLimit != nil {
+				c.OnRateLimit(wait)
+			}
+		case resp.StatusCode >= 500:
 			lastErr = &Error{resp.StatusCode, snippet(body)}
-			continue
-		}
-		if resp.StatusCode >= 300 {
+		case resp.StatusCode >= 300:
 			return &Error{resp.StatusCode, snippet(body)}
+		default:
+			if err := json.Unmarshal(body, out); err != nil {
+				return fmt.Errorf("unexpected answer from %s: %w", path, err)
+			}
+			return nil
 		}
-		if err := json.Unmarshal(body, out); err != nil {
-			return fmt.Errorf("unexpected answer from %s: %w", path, err)
+		if wait == 0 {
+			// network errors and server errors: a few tries with growing pauses
+			failures++
+			if failures > 6 {
+				return lastErr
+			}
+			wait = time.Duration(1<<failures) * time.Second
 		}
-		return nil
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(wait):
+		}
+	}
+}
+
+// retryAfter reads Tolgee's hint: {"retryAfter": ms} in the body or a Retry-After header in seconds.
+func retryAfter(resp *http.Response, body []byte) time.Duration {
+	var hint struct {
+		RetryAfter int64 `json:"retryAfter"`
+	}
+	if json.Unmarshal(body, &hint) == nil && hint.RetryAfter > 0 {
+		return time.Duration(hint.RetryAfter)*time.Millisecond + time.Second
+	}
+	if s, err := strconv.Atoi(resp.Header.Get("Retry-After")); err == nil && s > 0 {
+		return time.Duration(s)*time.Second + time.Second
 	}
-	return lastErr
+	return 30 * time.Second
 }
 
 func snippet(b []byte) string {

web/src/lib/jobs.svelte.ts

@@ -69,3 +69,9 @@ export function duration(seconds: number): string {
 	if (m < 60) return `${m} min ${seconds % 60}s`;
 	return `${Math.floor(m / 60)} h ${m % 60} min`;
 }
+
+/** Seconds the job is waiting for a remote rate limit, from its status line. */
+export function rateLimitWait(j: Job): number | null {
+	const m = /^rate_limited:(\d+)$/.exec(j.message);
+	return m && j.status === 'running' ? Number(m[1]) : null;
+}

web/src/lib/locales/en.json

@@ -134,6 +134,7 @@
   "jobs.kind.tolgee.import": "Tolgee import",
   "jobs.progress": "Progress",
   "jobs.progressLabel": "{done} of {total}",
+  "jobs.rateLimited": "Tolgee asked to slow down: waiting {time}, then it continues on its own",
   "jobs.result.mt": "{n} texts translated",
   "jobs.result.tolgee": "{keys} keys, {texts} texts imported",
   "jobs.retried": "Task queued again; it continues where it stopped",

web/src/lib/locales/pt-BR.json

@@ -134,6 +134,7 @@
   "jobs.kind.tolgee.import": "Importação do Tolgee",
   "jobs.progress": "Progresso",
   "jobs.progressLabel": "{done} de {total}",
+  "jobs.rateLimited": "O Tolgee pediu para ir mais devagar: aguardando {time} e seguindo sozinho",
   "jobs.result.mt": "{n} textos traduzidos",
   "jobs.result.tolgee": "{keys} chaves e {texts} textos importados",
   "jobs.retried": "Tarefa de volta na fila; ela continua de onde parou",

web/src/routes/(app)/jobs/+page.svelte

@@ -7,7 +7,7 @@
 	import { ListChecks, RotateCcw, X } from '@lucide/svelte';
 	import { onMount } from 'svelte';
 	import { api, type Job } from '#lib/api.ts';
-	import { watchJobs, isActive, percent, eta, duration } from '#lib/jobs.svelte.ts';
+	import { watchJobs, isActive, percent, eta, duration, rateLimitWait } from '#lib/jobs.svelte.ts';
 	import { t, num, relativeTime, formatDate } from '#lib/i18n.svelte.ts';
 
 	let jobs = $state<Job[] | null>(null);
@@ -78,6 +78,7 @@
 							<span class="meta lx-mono">
 								{num(j.done)} / {num(j.total)} · {percent(j)}%{#if left !== null} · {t('jobs.eta', { time: duration(left) })}{/if}
 							</span>
+							{#if rateLimitWait(j) !== null}<span class="meta">{t('jobs.rateLimited', { time: duration(rateLimitWait(j)!) })}</span>{/if}
 							{#if j.status === 'failed' && j.message}<span class="error">{j.message}</span>{/if}
 							{#if j.status === 'done' && summary(j)}<span class="meta">{summary(j)}</span>{/if}
 						</Stack>

web/src/routes/(app)/p/[slug]/+layout.svelte

@@ -1,7 +1,7 @@
 <!-- @codeatlasdev/ui demo #460 · MIT License (c) CodeAtlas · https://ui.codeatlas.com.br/docs/navigation/tabs#line -->
 <script lang="ts">
 	import { PageShell, Tabs, TabsList, TabsTrigger, EmptyState, Button, Chip, Row, Skeleton, Stack, Progress } from '@codeatlasdev/ui';
-	import { watchJobs, percent } from '#lib/jobs.svelte.ts';
+	import { watchJobs, percent, rateLimitWait, duration } from '#lib/jobs.svelte.ts';
 	import { session } from '#lib/session.svelte.ts';
 	import type { Job } from '#lib/api.ts';
 	import { FolderX } from '@lucide/svelte';
@@ -94,6 +94,7 @@
 				</Row>
 				<Progress value={percent(j)} label="{j.title} {percent(j)}%" />
 				<span class="lx-mono lx-muted small">{j.done.toLocaleString()} / {j.total.toLocaleString()} · {percent(j)}%</span>
+				{#if rateLimitWait(j) !== null}<span class="lx-muted small">{t('jobs.rateLimited', { time: duration(rateLimitWait(j)!) })}</span>{/if}
 			</div>
 		{/each}
 		{@render children()}