From 408cd6714f47ac57879f7bfa6a902d33670e1d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henrique=20Ara=C3=BAjo?= Date: Fri, 23 Oct 2026 15:37:05 -0300 Subject: [PATCH] perf: memory limit from the container, streamed exports, trigram search index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - At start, read the cgroup memory limit and keep the Go heap at 80% of it (GOMEMLIMIT still wins). The 256 MB container used to hit its limit thousands of times during a large export and get throttled. - Exports write each file straight to the response: flat and nested JSON are generated incrementally (byte-identical to encoding/json, tested), PO, Android, iOS and .properties write through a buffer. - Search: FTS5 trigram index with one row per key (name, description, every text). Writes only mark keys dirty; the index catches up in the background, and searches check dirty keys directly meanwhile, so results are always exact. Matches are confirmed with a Unicode case-insensitive contains (LIKE folds ASCII only, so 'AÇÃO' now finds 'ação'). - Migration 3 indexes existing keys on first start. Platify (160k keys, 5 languages) on the dev server: editor page 30 ms, search 127 ms (was 2.5 s), full ZIP export 6.6 s (was 19 s), 91 MB peak RAM; the search index adds 54 MB to the database. --- cmd/lowlex/main.go | 1 + cmd/lowlex/memlimit.go | 35 ++++++ internal/formats/formats.go | 79 +----------- internal/formats/formats_test.go | 8 -- internal/formats/write.go | 204 +++++++++++++++++++++++++++++++ internal/formats/write_test.go | 61 +++++++++ internal/server/io.go | 20 ++- internal/server/jobs.go | 5 +- internal/server/keys.go | 21 +++- internal/server/load_test.go | 23 +++- internal/server/search.go | 104 ++++++++++++++++ internal/server/search_test.go | 61 +++++++++ internal/store/schema3.sql | 32 +++++ internal/store/store.go | 25 +++- 14 files changed, 574 insertions(+), 105 deletions(-) create mode 100644 cmd/lowlex/memlimit.go create mode 100644 internal/formats/write.go create mode 100644 internal/formats/write_test.go create mode 100644 internal/server/search.go create mode 100644 internal/server/search_test.go create mode 100644 internal/store/schema3.sql diff --git a/cmd/lowlex/main.go b/cmd/lowlex/main.go index 19d6da2..e9ddf16 100644 --- a/cmd/lowlex/main.go +++ b/cmd/lowlex/main.go @@ -84,6 +84,7 @@ func main() { } func serve(cfg config.Config) error { + setMemoryLimit() db, err := store.Open(cfg.DataDir) if err != nil { return err diff --git a/cmd/lowlex/memlimit.go b/cmd/lowlex/memlimit.go new file mode 100644 index 0000000..5cf3ae9 --- /dev/null +++ b/cmd/lowlex/memlimit.go @@ -0,0 +1,35 @@ +package main + +import ( + "log" + "os" + "runtime/debug" + "strconv" + "strings" +) + +// setMemoryLimit keeps the Go heap under the container's memory limit. Without it the +// garbage collector does not know about the cgroup limit, lets the heap grow to it and the +// kernel then throttles the process. GOMEMLIMIT, when set, wins. +func setMemoryLimit() { + if os.Getenv("GOMEMLIMIT") != "" { + return + } + for _, path := range []string{ + "/sys/fs/cgroup/memory.max", // cgroup v2 + "/sys/fs/cgroup/memory/memory.limit_in_bytes", // cgroup v1 + } { + b, err := os.ReadFile(path) + if err != nil { + continue + } + limit, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64) + if err != nil || limit <= 0 || limit > 1<<50 { // "max" or the v1 "unlimited" value + return + } + soft := limit / 10 * 8 + debug.SetMemoryLimit(soft) + log.Printf("memory limit %d MB: keeping the heap under %d MB", limit>>20, soft>>20) + return + } +} diff --git a/internal/formats/formats.go b/internal/formats/formats.go index 74a0f02..49707e8 100644 --- a/internal/formats/formats.go +++ b/internal/formats/formats.go @@ -90,39 +90,6 @@ func Parse(format string, data []byte) (map[string]string, error) { return nil, fmt.Errorf("unsupported format %q", format) } -// Render writes a single-language file. -func Render(format string, entries []Entry) ([]byte, error) { - sort.Slice(entries, func(i, j int) bool { return entries[i].Key < entries[j].Key }) - switch format { - case "json": - m := make(map[string]string, len(entries)) - for _, e := range entries { - m[e.Key] = e.Text - } - return marshal(m) - case "json-nested": - return marshal(nest(entries)) - case "po": - return renderPO(entries), nil - case "android": - return renderAndroid(entries), nil - case "ios": - return renderStrings(entries), nil - case "properties": - return renderProperties(entries), nil - } - return nil, fmt.Errorf("unsupported format %q", format) -} - -func marshal(v any) ([]byte, error) { - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.SetEscapeHTML(false) - enc.SetIndent("", " ") - err := enc.Encode(v) - return buf.Bytes(), err -} - // --- JSON --- func parseJSON(data []byte) (map[string]string, error) { @@ -166,36 +133,6 @@ func flatten(prefix string, v any, out map[string]string) { } } -func nest(entries []Entry) map[string]any { - root := map[string]any{} - for _, e := range entries { - parts := strings.Split(e.Key, ".") - node := root - for i, p := range parts { - if i == len(parts)-1 { - if _, isObj := node[p].(map[string]any); isObj { - // "a" and "a.b" both exist: keep "a" next to its children with the full key - root[e.Key] = e.Text - } else { - node[p] = e.Text - } - break - } - next, ok := node[p].(map[string]any) - if !ok { - if _, isLeaf := node[p]; isLeaf { - node[strings.Join(parts[i:], ".")] = e.Text - break - } - next = map[string]any{} - node[p] = next - } - node = next - } - } - return root -} - // --- CSV: key, description, then one column per language --- // ParseCSV returns texts by language code then key. @@ -325,8 +262,7 @@ func poUnquote(s string) string { return b.String() } -func renderPO(entries []Entry) []byte { - var b bytes.Buffer +func renderPO(b *bufio.Writer, entries []Entry) { b.WriteString("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n\n") for _, e := range entries { if e.Description != "" { @@ -336,7 +272,6 @@ func renderPO(entries []Entry) []byte { } b.WriteString("msgid " + poQuote(e.Key) + "\nmsgstr " + poQuote(e.Text) + "\n\n") } - return b.Bytes() } func parsePO(data []byte) (map[string]string, error) { @@ -460,8 +395,7 @@ func androidUnescape(s string) string { return b.String() } -func renderAndroid(entries []Entry) []byte { - var b bytes.Buffer +func renderAndroid(b *bufio.Writer, entries []Entry) { b.WriteString("\n\n") for _, e := range entries { name := strings.NewReplacer(".", "_", "-", "_", " ", "_").Replace(e.Key) @@ -471,7 +405,6 @@ func renderAndroid(entries []Entry) []byte { b.WriteString(` ` + androidEscape(e.Text) + "\n") } b.WriteString("\n") - return b.Bytes() } func parseAndroid(data []byte) (map[string]string, error) { @@ -507,15 +440,13 @@ func stringsQuote(s string) string { return `"` + r.Replace(s) + `"` } -func renderStrings(entries []Entry) []byte { - var b bytes.Buffer +func renderStrings(b *bufio.Writer, entries []Entry) { for _, e := range entries { if e.Description != "" { b.WriteString("/* " + strings.ReplaceAll(e.Description, "*/", "* /") + " */\n") } b.WriteString(stringsQuote(e.Key) + " = " + stringsQuote(e.Text) + ";\n") } - return b.Bytes() } var stringsLine = regexp.MustCompile(`(?s)"((?:[^"\\]|\\.)*)"\s*=\s*"((?:[^"\\]|\\.)*)"\s*;`) @@ -561,8 +492,7 @@ func utf16ToUTF8(b []byte) []byte { // --- Java .properties (UTF-8) --- -func renderProperties(entries []Entry) []byte { - var b bytes.Buffer +func renderProperties(b *bufio.Writer, entries []Entry) { esc := strings.NewReplacer(`\`, `\\`, "\n", `\n`, "\t", `\t`, "=", `\=`, ":", `\:`) for _, e := range entries { if e.Description != "" { @@ -570,7 +500,6 @@ func renderProperties(entries []Entry) []byte { } b.WriteString(esc.Replace(e.Key) + "=" + strings.NewReplacer(`\`, `\\`, "\n", `\n`, "\t", `\t`).Replace(e.Text) + "\n") } - return b.Bytes() } func parseProperties(data []byte) (map[string]string, error) { diff --git a/internal/formats/formats_test.go b/internal/formats/formats_test.go index b806fd3..e45efb5 100644 --- a/internal/formats/formats_test.go +++ b/internal/formats/formats_test.go @@ -49,11 +49,3 @@ func TestCSV(t *testing.T) { } } -func TestNestConflict(t *testing.T) { - out := nest([]Entry{{Key: "a", Text: "A"}, {Key: "a.b", Text: "AB"}}) - flat := map[string]string{} - flatten("", out, flat) - if flat["a"] != "A" || flat["a.b"] != "AB" { - t.Fatalf("conflict lost data: %v", flat) - } -} diff --git a/internal/formats/write.go b/internal/formats/write.go new file mode 100644 index 0000000..53c156a --- /dev/null +++ b/internal/formats/write.go @@ -0,0 +1,204 @@ +package formats + +import ( + "bufio" + "bytes" + "fmt" + "io" + "sort" + "strings" + "unicode/utf8" +) + +// Render returns a single-language file. Write streams it instead. +func Render(format string, entries []Entry) ([]byte, error) { + var buf bytes.Buffer + err := Write(&buf, format, entries) + return buf.Bytes(), err +} + +// Write streams a single-language file to w without building it in memory first. +// entries is sorted in place. +func Write(w io.Writer, format string, entries []Entry) error { + bw := bufio.NewWriterSize(w, 64<<10) + switch format { + case "json": + sortByKey(entries) + writeFlatJSON(bw, entries) + case "json-nested": + writeNestedJSON(bw, entries) + case "po": + sortByKey(entries) + renderPO(bw, entries) + case "android": + sortByKey(entries) + renderAndroid(bw, entries) + case "ios": + sortByKey(entries) + renderStrings(bw, entries) + case "properties": + sortByKey(entries) + renderProperties(bw, entries) + default: + return fmt.Errorf("unsupported format %q", format) + } + return bw.Flush() +} + +func sortByKey(entries []Entry) { + sort.Slice(entries, func(i, j int) bool { return entries[i].Key < entries[j].Key }) +} + +// writeJSONString writes s as a JSON string, escaped like encoding/json without HTML escaping. +func writeJSONString(w *bufio.Writer, s string) { + const hex = "0123456789abcdef" + w.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + c := s[i] + if c < utf8.RuneSelf { + if c >= 0x20 && c != '"' && c != '\\' { + i++ + continue + } + w.WriteString(s[start:i]) + switch c { + case '"', '\\': + w.WriteByte('\\') + w.WriteByte(c) + case '\n': + w.WriteString(`\n`) + case '\r': + w.WriteString(`\r`) + case '\t': + w.WriteString(`\t`) + default: + w.WriteString(`\u00`) + w.WriteByte(hex[c>>4]) + w.WriteByte(hex[c&0xf]) + } + i++ + start = i + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + switch { + case r == utf8.RuneError && size == 1: + w.WriteString(s[start:i]) + w.WriteString(`\u` + "fffd") // the JSON escape for U+FFFD, as encoding/json writes it + case r == '
' || r == '
': + w.WriteString(s[start:i]) + w.WriteString(`\u202`) + w.WriteByte(hex[r&0xf]) + default: + i += size + continue + } + i += size + start = i + } + w.WriteString(s[start:]) + w.WriteByte('"') +} + +// writeFlatJSON writes {"key": "text", …}, indented with two spaces like json.MarshalIndent. +func writeFlatJSON(w *bufio.Writer, entries []Entry) { + if len(entries) == 0 { + w.WriteString("{}\n") + return + } + w.WriteString("{\n") + for i, e := range entries { + w.WriteString(" ") + writeJSONString(w, e.Key) + w.WriteString(": ") + writeJSONString(w, e.Text) + if i < len(entries)-1 { + w.WriteByte(',') + } + w.WriteByte('\n') + } + w.WriteString("}\n") +} + +// nestedPath splits a key on dots. When a shorter key is itself a text ("a" and "a.b"), +// the rest of the longer key stays joined at that level: {"a": "A", "a.b": "AB"}. +func nestedPath(key string, isKey map[string]bool) []string { + parts := strings.Split(key, ".") + for i := 1; i < len(parts); i++ { + if isKey[strings.Join(parts[:i], ".")] { + return append(parts[:i-1:i-1], strings.Join(parts[i-1:], ".")) + } + } + return parts +} + +// writeNestedJSON streams {"a": {"b": "text"}}, walking the keys sorted by path segment. +func writeNestedJSON(w *bufio.Writer, entries []Entry) { + if len(entries) == 0 { + w.WriteString("{}\n") + return + } + isKey := make(map[string]bool, len(entries)) + for _, e := range entries { + isKey[e.Key] = true + } + type item struct { + path []string + text string + } + items := make([]item, len(entries)) + for i, e := range entries { + items[i] = item{nestedPath(e.Key, isKey), e.Text} + } + sort.Slice(items, func(i, j int) bool { + a, b := items[i].path, items[j].path + for k := 0; k < len(a) && k < len(b); k++ { + if a[k] != b[k] { + return a[k] < b[k] + } + } + return len(a) < len(b) + }) + + indent := func(depth int) { + for range depth { + w.WriteString(" ") + } + } + w.WriteString("{\n") + var open []string // path of the objects currently open below the root + for n, it := range items { + parent := it.path[:len(it.path)-1] + common := 0 + for common < len(open) && common < len(parent) && open[common] == parent[common] { + common++ + } + for len(open) > common { // close the objects this key is not in + w.WriteByte('\n') + indent(len(open)) + w.WriteByte('}') + open = open[:len(open)-1] + } + if n > 0 { + w.WriteString(",\n") + } + for _, seg := range parent[common:] { // open the objects it needs + indent(len(open) + 1) + writeJSONString(w, seg) + w.WriteString(": {\n") + open = append(open, seg) + } + indent(len(open) + 1) + writeJSONString(w, it.path[len(it.path)-1]) + w.WriteString(": ") + writeJSONString(w, it.text) + } + for len(open) > 0 { + w.WriteByte('\n') + indent(len(open)) + w.WriteByte('}') + open = open[:len(open)-1] + } + w.WriteString("\n}\n") +} diff --git a/internal/formats/write_test.go b/internal/formats/write_test.go new file mode 100644 index 0000000..43f297e --- /dev/null +++ b/internal/formats/write_test.go @@ -0,0 +1,61 @@ +package formats + +import ( + "bytes" + "encoding/json" + "fmt" + "testing" +) + +func TestNestConflict(t *testing.T) { + data, err := Render("json-nested", []Entry{{Key: "a", Text: "A"}, {Key: "a.b", Text: "AB"}, {Key: "a.b.c", Text: "ABC"}, {Key: "x.y", Text: "XY"}}) + if err != nil { + t.Fatal(err) + } + flat, err := Parse("json", data) + if err != nil { + t.Fatalf("%v\n%s", err, data) + } + if flat["a"] != "A" || flat["a.b"] != "AB" || flat["a.b.c"] != "ABC" || flat["x.y"] != "XY" || len(flat) != 4 { + t.Fatalf("conflict lost data: %v\n%s", flat, data) + } +} + +// The streamed writers must produce exactly what encoding/json would. +func TestJSONMatchesEncodingJSON(t *testing.T) { + texts := []string{"plain", `quote " and \ backslash`, "line\nbreak\ttab\r", "ctrl \x01\x1f", "&", + "emoji 🎉 ção 日本", "sep 
 
", "bad \xff utf8", ""} + var entries []Entry + flatWant := map[string]string{} + nestWant := map[string]any{} + grp := map[string]any{} + for i, tx := range texts { + k := fmt.Sprintf("k%d", i) + entries = append(entries, Entry{Key: k, Text: tx}, Entry{Key: "grp." + k, Text: tx}, Entry{Key: "a.b.c." + k, Text: tx}) + flatWant[k], flatWant["grp."+k], flatWant["a.b.c."+k] = tx, tx, tx + nestWant[k] = tx + grp[k] = tx + } + nestWant["grp"] = grp + nestWant["a"] = map[string]any{"b": map[string]any{"c": grp}} + + enc := func(v any) string { + var b bytes.Buffer + e := json.NewEncoder(&b) + e.SetEscapeHTML(false) + e.SetIndent("", " ") + e.Encode(v) + return b.String() + } + got, _ := Render("json", append([]Entry(nil), entries...)) + if want := enc(flatWant); string(got) != want { + t.Fatalf("flat JSON differs\ngot:\n%s\nwant:\n%s", got, want) + } + got, _ = Render("json-nested", append([]Entry(nil), entries...)) + if want := enc(nestWant); string(got) != want { + t.Fatalf("nested JSON differs\ngot:\n%s\nwant:\n%s", got, want) + } + if got, _ := Render("json-nested", nil); string(got) != "{}\n" { + t.Fatalf("empty: %q", got) + } +} diff --git a/internal/server/io.go b/internal/server/io.go index 53052f1..1359d66 100644 --- a/internal/server/io.go +++ b/internal/server/io.go @@ -387,7 +387,7 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx) return err } } - render := func(texts map[int64]string, namespace string) ([]byte, error) { + entriesFor := func(texts map[int64]string, namespace string) []formats.Entry { entries := []formats.Entry{} for _, k := range keys { if k.ns != namespace { @@ -402,7 +402,7 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx) } entries = append(entries, formats.Entry{Key: k.name, Text: text, Description: k.desc}) } - return formats.Render(format, entries) + return entries } namespaces := keyNamespaces(keys) @@ -415,14 +415,12 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx) if err != nil { return err } - body, err := render(texts, nsName) - if err != nil { - return err - } - return sendFile(w, exportName(info, codes[0], ""), contentType(info.Ext), body) + w.Header().Set("Content-Type", contentType(info.Ext)) + w.Header().Set("Content-Disposition", `attachment; filename="`+path.Base(exportName(info, codes[0], ""))+`"`) + return formats.Write(w, format, entriesFor(texts, nsName)) } - // the ZIP is written straight to the response + // the ZIP is written straight to the response, one file at a time w.Header().Set("Content-Type", "application/zip") w.Header().Set("Content-Disposition", `attachment; filename="`+base+"-"+format+`.zip"`) zw := zip.NewWriter(w) @@ -432,15 +430,11 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx) return err } for _, n := range namespaces { - body, err := render(texts, n) - if err != nil { - return err - } f, err := zw.Create(exportName(info, code, n)) if err != nil { return err } - if _, err := f.Write(body); err != nil { + if err := formats.Write(f, format, entriesFor(texts, n)); err != nil { return err } } diff --git a/internal/server/jobs.go b/internal/server/jobs.go index 4027b0e..c57e4ff 100644 --- a/internal/server/jobs.go +++ b/internal/server/jobs.go @@ -28,6 +28,7 @@ var jobHandlers = map[string]jobHandler{} type jobRunner struct { wake chan struct{} + ctx context.Context stop context.CancelFunc wg sync.WaitGroup mu sync.Mutex @@ -50,9 +51,10 @@ type jobRun struct { func (s *Server) startJobs() { ctx, cancel := context.WithCancel(context.Background()) - s.jobs = &jobRunner{wake: make(chan struct{}, 1), stop: cancel, running: map[int64]context.CancelFunc{}} + s.jobs = &jobRunner{wake: make(chan struct{}, 1), ctx: ctx, stop: cancel, running: map[int64]context.CancelFunc{}} // jobs left running by a previous process resume from their saved state s.db.Exec(`UPDATE jobs SET status = 'queued' WHERE status = 'running'`) + s.flushSearchLater() // keys from an upgrade or an interrupted flush for range jobWorkers { s.jobs.wg.Add(1) go s.jobWorker(ctx) @@ -151,6 +153,7 @@ func (s *Server) runJob(parent context.Context, j *jobRun) { switch { case err == nil: s.finishJob(j.ID, "done", "", result) + s.flushSearchLater() // imports change many keys; index them now rather than on the next search case errors.Is(err, errJobCancelled): s.finishJob(j.ID, "cancelled", "", result) case parent.Err() != nil: diff --git a/internal/server/keys.go b/internal/server/keys.go index c8d912c..ff4d56c 100644 --- a/internal/server/keys.go +++ b/internal/server/keys.go @@ -53,10 +53,23 @@ func (s *Server) listKeys(w http.ResponseWriter, r *http.Request, pc *projectCtx args = append(args, ns[0]) } if term := strings.TrimSpace(q.Get("q")); term != "" { - like := likeArg(term) - where = append(where, `(k.name LIKE ? ESCAPE '\' OR k.description LIKE ? ESCAPE '\' - OR EXISTS (SELECT 1 FROM translations t WHERE t.key_id = k.id AND t.text LIKE ? ESCAPE '\'))`) - args = append(args, like, like, like) + if fts := trigramQuery(term); fts != "" { + // the trigram index narrows the keys down (keys changed since the last index update + // are checked directly), then a case-insensitive substring test in every script keeps + // only real matches. The index catches up in the background. + s.flushSearchLater() + needle := strings.ToLower(term) + where = append(where, `k.id IN (SELECT rowid FROM search_index WHERE search_index MATCH ? UNION ALL SELECT key_id FROM search_dirty) + AND (lowlex_contains(k.name, ?) OR lowlex_contains(k.description, ?) + OR EXISTS (SELECT 1 FROM translations t WHERE t.key_id = k.id AND lowlex_contains(t.text, ?)))`) + args = append(args, fts, needle, needle, needle) + } else { + // one or two characters: too short for trigrams, scan with LIKE + like := likeArg(term) + where = append(where, `(k.name LIKE ? ESCAPE '\' OR k.description LIKE ? ESCAPE '\' + OR EXISTS (SELECT 1 FROM translations t WHERE t.key_id = k.id AND t.text LIKE ? ESCAPE '\'))`) + args = append(args, like, like, like) + } } if tag := strings.TrimSpace(q.Get("tag")); tag != "" { where = append(where, "EXISTS (SELECT 1 FROM json_each(k.tags) WHERE value = ?)") diff --git a/internal/server/load_test.go b/internal/server/load_test.go index 6cff0e2..f8c77d3 100644 --- a/internal/server/load_test.go +++ b/internal/server/load_test.go @@ -52,8 +52,27 @@ func TestTolgeeImportLoad(t *testing.T) { t.Logf("%d keys imported in %s (%.0f keys/s); slowest key list during import %s; heap in use %d MB", n, took.Round(time.Millisecond), float64(n)/took.Seconds(), worst.Round(time.Millisecond), mem.HeapInuse>>20) - // a page of the editor, a search and the project stats on the full project - for _, path := range []string{"/api/projects/big-app/keys?per=50&page=1500", "/api/projects/big-app/keys?q=Texto+15999", "/api/projects/big-app", "/api/projects"} { + // the first search waits for the index of everything just imported + t0 := time.Now() + admin.do("GET", "/api/projects/big-app/keys?q=Texto+15999", nil, 200) + t.Logf("first search (indexes %d keys): %s", n, time.Since(t0).Round(time.Millisecond)) + + // searches stay correct while the index catches up in the background; then they are fast + t0 = time.Now() + for time.Since(t0) < time.Minute { + t1 := time.Now() + admin.do("GET", "/api/projects/big-app/keys?q=xto+1234", nil, 200) + if time.Since(t1) < 150*time.Millisecond { + break + } + time.Sleep(time.Second) + } + time.Sleep(10 * time.Second) // let the last index batches and the WAL checkpoint finish + t.Logf("index caught up after %s", time.Since(t0).Round(time.Second)) + + // a page of the editor, searches and the project stats on the full project + for _, path := range []string{"/api/projects/big-app/keys?per=50&page=1500", "/api/projects/big-app/keys?q=Texto+15999", + "/api/projects/big-app/keys?q=xto+1234", "/api/projects/big-app/keys?q=ctx", "/api/projects/big-app/keys?q=7", "/api/projects/big-app", "/api/projects"} { t0 := time.Now() admin.do("GET", path, nil, 200) t.Logf("GET %s: %s", path, time.Since(t0).Round(time.Millisecond)) diff --git a/internal/server/search.go b/internal/server/search.go new file mode 100644 index 0000000..6e3a8f1 --- /dev/null +++ b/internal/server/search.go @@ -0,0 +1,104 @@ +package server + +import ( + "context" + "log" + "strings" + "sync" + "sync/atomic" +) + +// Search uses an FTS5 trigram index with one row per key (see store/schema3.sql). Writes +// mark keys dirty; flushSearch folds them in the background, so the index never slows down +// writing, and searches check dirty keys directly until then. + +var searchMu sync.Mutex + +const searchBatch = 1000 + +// flushSearch indexes every key marked dirty since the last flush. +func (s *Server) flushSearch(ctx context.Context) error { + searchMu.Lock() + defer searchMu.Unlock() + for { + var pending int + if err := s.db.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM search_dirty)`).Scan(&pending); err != nil || pending == 0 { + return err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + rows, err := tx.QueryContext(ctx, `SELECT key_id FROM search_dirty LIMIT ?`, searchBatch) + if err != nil { + tx.Rollback() + return err + } + var ids []int64 + for rows.Next() { + var id int64 + rows.Scan(&id) + ids = append(ids, id) + } + rows.Close() + for _, id := range ids { + if _, err := tx.ExecContext(ctx, `DELETE FROM search_index WHERE rowid = ?`, id); err != nil { + tx.Rollback() + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO search_index (rowid, text) + SELECT k.id, k.name || char(10) || k.description || char(10) || + COALESCE((SELECT group_concat(t.text, char(10)) FROM translations t WHERE t.key_id = k.id), '') + FROM keys k WHERE k.id = ?`, id); err != nil { + tx.Rollback() + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM search_dirty WHERE key_id = ?`, id); err != nil { + tx.Rollback() + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + } +} + +var flushing atomic.Bool + +// flushSearchLater indexes dirty keys in the background, e.g. after an edit or a large import. +// One flush runs at a time; Close waits for it. +func (s *Server) flushSearchLater() { + if !flushing.CompareAndSwap(false, true) { + return + } + s.jobs.wg.Add(1) + go func() { + defer s.jobs.wg.Done() + defer flushing.Store(false) + if err := s.flushSearch(s.jobs.ctx); err != nil && s.jobs.ctx.Err() == nil { + log.Printf("search index: %v", err) + } + }() +} + +// trigramQuery turns a search term into an FTS5 query that matches every text containing it: +// all of its trigrams must be present. Terms shorter than three characters cannot use the +// index and return "". +func trigramQuery(term string) string { + r := []rune(strings.ToLower(term)) + if len(r) < 3 { + return "" + } + seen := map[string]bool{} + var parts []string + for i := 0; i+3 <= len(r); i++ { + g := string(r[i : i+3]) + if seen[g] { + continue + } + seen[g] = true + parts = append(parts, `"`+strings.ReplaceAll(g, `"`, `""`)+`"`) + } + return strings.Join(parts, " AND ") +} diff --git a/internal/server/search_test.go b/internal/server/search_test.go new file mode 100644 index 0000000..3793cb9 --- /dev/null +++ b/internal/server/search_test.go @@ -0,0 +1,61 @@ +package server + +import ( + "fmt" + "net/url" + "testing" +) + +func TestSearch(t *testing.T) { + _, newClient := newTestServer(t) + admin := newClient() + admin.do("POST", "/api/auth/login", map[string]string{"username": "admin", "password": DefaultAdminPassword}, 200) + admin.do("POST", "/api/auth/password", map[string]string{"current_password": DefaultAdminPassword, "new_password": "search-test-password"}, 200) + admin.do("POST", "/api/projects", map[string]any{"name": "App", "languages": []string{"pt-BR"}}, 201) + admin.upload("/api/projects/app/import", "en.json", `{"menu.save":"Save changes","menu.quit":"Quit the game","ach.desc":""}`, nil, 200) + admin.upload("/api/projects/app/import", "pt-BR.json", `{"menu.save":"Salvar alterações","menu.quit":"Sair do jogo"}`, nil, 200) + k := admin.do("POST", "/api/projects/app/keys", map[string]any{"name": "dialog.title", "description": "Shown when the dragon appears"}, 201) + dialogID := int(k["key"].(map[string]any)["id"].(float64)) + + names := func(q string) map[string]bool { + t.Helper() + res := admin.do("GET", "/api/projects/app/keys?q="+url.QueryEscape(q), nil, 200) + out := map[string]bool{} + for _, k := range res["keys"].([]any) { + out[k.(map[string]any)["name"].(string)] = true + } + if int(res["total"].(float64)) != len(out) { + t.Fatalf("q=%q: total %v, keys %v", q, res["total"], out) + } + return out + } + expect := func(q string, want ...string) { + t.Helper() + got := names(q) + if len(got) != len(want) { + t.Fatalf("q=%q: got %v, want %v", q, got, want) + } + for _, w := range want { + if !got[w] { + t.Fatalf("q=%q: got %v, want %v", q, got, want) + } + } + } + + expect("change", "menu.save") // inside a word + expect("ALTERAÇ", "menu.save") // accents, upper case (FTS) then LIKE confirms + expect("menu.qu", "menu.quit") // key name + expect("dragon", "dialog.title") // description + expect("jo", "menu.quit") // two letters: no index, plain LIKE + expect("e", "menu.save", "menu.quit", "dialog.title", "ach.desc") // one letter, key names count + expect("nothing like this") + + // edits reach the index on the next search + admin.do("PUT", fmt.Sprintf("/api/projects/app/translations/%d/en", dialogID), map[string]any{"text": "A wild dragon appears"}, 200) + expect("wild drag", "dialog.title") + admin.do("PUT", fmt.Sprintf("/api/projects/app/translations/%d/en", dialogID), map[string]any{"text": "A calm sea"}, 200) + expect("wild drag") + admin.do("DELETE", fmt.Sprintf("/api/projects/app/keys/%d", dialogID), nil, 200) + expect("dragon") + expect("calm sea") +} diff --git a/internal/store/schema3.sql b/internal/store/schema3.sql new file mode 100644 index 0000000..fad9b15 --- /dev/null +++ b/internal/store/schema3.sql @@ -0,0 +1,32 @@ +-- LowLex schema, version 3: full-text search. +-- One row per key (rowid = key id) with its name, description and every text, split into +-- trigrams so any substring of 3+ characters can be found without scanning every text. +-- Writes only mark the key in search_dirty (with WHERE NOT EXISTS: inside a trigger, OR IGNORE +-- would take the outer statement's conflict policy and fail during upserts); the server folds dirty keys into the index in +-- batches, so imports stay fast. + +CREATE VIRTUAL TABLE search_index USING fts5(text, content='', contentless_delete=1, tokenize='trigram', detail='none'); + +CREATE TABLE search_dirty (key_id INTEGER PRIMARY KEY); + +CREATE TRIGGER search_translation_insert AFTER INSERT ON translations BEGIN + INSERT INTO search_dirty (key_id) SELECT new.key_id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = new.key_id); +END; +CREATE TRIGGER search_translation_update AFTER UPDATE OF text ON translations BEGIN + INSERT INTO search_dirty (key_id) SELECT new.key_id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = new.key_id); +END; +CREATE TRIGGER search_translation_delete AFTER DELETE ON translations BEGIN + INSERT INTO search_dirty (key_id) SELECT old.key_id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = old.key_id); +END; +CREATE TRIGGER search_key_insert AFTER INSERT ON keys BEGIN + INSERT INTO search_dirty (key_id) SELECT new.id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = new.id); +END; +CREATE TRIGGER search_key_update AFTER UPDATE OF name, description ON keys BEGIN + INSERT INTO search_dirty (key_id) SELECT new.id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = new.id); +END; +CREATE TRIGGER search_key_delete AFTER DELETE ON keys BEGIN + INSERT INTO search_dirty (key_id) SELECT old.id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = old.id); +END; + +-- existing keys are indexed on the first start after the upgrade +INSERT INTO search_dirty (key_id) SELECT id FROM keys; diff --git a/internal/store/store.go b/internal/store/store.go index 436ea6d..7a0bbba 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -3,22 +3,43 @@ package store import ( "database/sql" + "database/sql/driver" _ "embed" "fmt" "os" "path/filepath" + "strings" - _ "modernc.org/sqlite" + "modernc.org/sqlite" ) +// lowlex_contains(text, needle) is a substring test that ignores case in every script, +// unlike LIKE, which folds ASCII only. needle must already be lower case. +func init() { + sqlite.MustRegisterDeterministicScalarFunction("lowlex_contains", 2, func(_ *sqlite.FunctionContext, args []driver.Value) (driver.Value, error) { + text, _ := args[0].(string) + needle, _ := args[1].(string) + if b, ok := args[0].([]byte); ok { + text = string(b) + } + if strings.Contains(strings.ToLower(text), needle) { + return int64(1), nil + } + return int64(0), nil + }) +} + //go:embed schema.sql var schemaV1 string //go:embed schema2.sql var schemaV2 string +//go:embed schema3.sql +var schemaV3 string + // migrations[i] moves the database from user_version i to i+1. -var migrations = []string{schemaV1, schemaV2} +var migrations = []string{schemaV1, schemaV2, schemaV3} // Open creates the data directory if needed, opens lowlex.db and migrates it. func Open(dataDir string) (*sql.DB, error) {