diff --git a/cmd/lowlex/main.go b/cmd/lowlex/main.go
index 19d6da2..e9ddf16 100644
--- a/cmd/lowlex/main.go
+++ b/cmd/lowlex/main.go
@@ -84,6 +84,7 @@ func main() {
}
func serve(cfg config.Config) error {
+ setMemoryLimit()
db, err := store.Open(cfg.DataDir)
if err != nil {
return err
diff --git a/cmd/lowlex/memlimit.go b/cmd/lowlex/memlimit.go
new file mode 100644
index 0000000..5cf3ae9
--- /dev/null
+++ b/cmd/lowlex/memlimit.go
@@ -0,0 +1,35 @@
+package main
+
+import (
+ "log"
+ "os"
+ "runtime/debug"
+ "strconv"
+ "strings"
+)
+
+// setMemoryLimit keeps the Go heap under the container's memory limit. Without it the
+// garbage collector does not know about the cgroup limit, lets the heap grow to it and the
+// kernel then throttles the process. GOMEMLIMIT, when set, wins.
+func setMemoryLimit() {
+ if os.Getenv("GOMEMLIMIT") != "" {
+ return
+ }
+ for _, path := range []string{
+ "/sys/fs/cgroup/memory.max", // cgroup v2
+ "/sys/fs/cgroup/memory/memory.limit_in_bytes", // cgroup v1
+ } {
+ b, err := os.ReadFile(path)
+ if err != nil {
+ continue
+ }
+ limit, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
+ if err != nil || limit <= 0 || limit > 1<<50 { // "max" or the v1 "unlimited" value
+ return
+ }
+ soft := limit / 10 * 8
+ debug.SetMemoryLimit(soft)
+ log.Printf("memory limit %d MB: keeping the heap under %d MB", limit>>20, soft>>20)
+ return
+ }
+}
diff --git a/internal/formats/formats.go b/internal/formats/formats.go
index 74a0f02..49707e8 100644
--- a/internal/formats/formats.go
+++ b/internal/formats/formats.go
@@ -90,39 +90,6 @@ func Parse(format string, data []byte) (map[string]string, error) {
return nil, fmt.Errorf("unsupported format %q", format)
}
-// Render writes a single-language file.
-func Render(format string, entries []Entry) ([]byte, error) {
- sort.Slice(entries, func(i, j int) bool { return entries[i].Key < entries[j].Key })
- switch format {
- case "json":
- m := make(map[string]string, len(entries))
- for _, e := range entries {
- m[e.Key] = e.Text
- }
- return marshal(m)
- case "json-nested":
- return marshal(nest(entries))
- case "po":
- return renderPO(entries), nil
- case "android":
- return renderAndroid(entries), nil
- case "ios":
- return renderStrings(entries), nil
- case "properties":
- return renderProperties(entries), nil
- }
- return nil, fmt.Errorf("unsupported format %q", format)
-}
-
-func marshal(v any) ([]byte, error) {
- var buf bytes.Buffer
- enc := json.NewEncoder(&buf)
- enc.SetEscapeHTML(false)
- enc.SetIndent("", " ")
- err := enc.Encode(v)
- return buf.Bytes(), err
-}
-
// --- JSON ---
func parseJSON(data []byte) (map[string]string, error) {
@@ -166,36 +133,6 @@ func flatten(prefix string, v any, out map[string]string) {
}
}
-func nest(entries []Entry) map[string]any {
- root := map[string]any{}
- for _, e := range entries {
- parts := strings.Split(e.Key, ".")
- node := root
- for i, p := range parts {
- if i == len(parts)-1 {
- if _, isObj := node[p].(map[string]any); isObj {
- // "a" and "a.b" both exist: keep "a" next to its children with the full key
- root[e.Key] = e.Text
- } else {
- node[p] = e.Text
- }
- break
- }
- next, ok := node[p].(map[string]any)
- if !ok {
- if _, isLeaf := node[p]; isLeaf {
- node[strings.Join(parts[i:], ".")] = e.Text
- break
- }
- next = map[string]any{}
- node[p] = next
- }
- node = next
- }
- }
- return root
-}
-
// --- CSV: key, description, then one column per language ---
// ParseCSV returns texts by language code then key.
@@ -325,8 +262,7 @@ func poUnquote(s string) string {
return b.String()
}
-func renderPO(entries []Entry) []byte {
- var b bytes.Buffer
+func renderPO(b *bufio.Writer, entries []Entry) {
b.WriteString("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n\n")
for _, e := range entries {
if e.Description != "" {
@@ -336,7 +272,6 @@ func renderPO(entries []Entry) []byte {
}
b.WriteString("msgid " + poQuote(e.Key) + "\nmsgstr " + poQuote(e.Text) + "\n\n")
}
- return b.Bytes()
}
func parsePO(data []byte) (map[string]string, error) {
@@ -460,8 +395,7 @@ func androidUnescape(s string) string {
return b.String()
}
-func renderAndroid(entries []Entry) []byte {
- var b bytes.Buffer
+func renderAndroid(b *bufio.Writer, entries []Entry) {
b.WriteString("\n\n")
for _, e := range entries {
name := strings.NewReplacer(".", "_", "-", "_", " ", "_").Replace(e.Key)
@@ -471,7 +405,6 @@ func renderAndroid(entries []Entry) []byte {
b.WriteString(` ` + androidEscape(e.Text) + "\n")
}
b.WriteString("\n")
- return b.Bytes()
}
func parseAndroid(data []byte) (map[string]string, error) {
@@ -507,15 +440,13 @@ func stringsQuote(s string) string {
return `"` + r.Replace(s) + `"`
}
-func renderStrings(entries []Entry) []byte {
- var b bytes.Buffer
+func renderStrings(b *bufio.Writer, entries []Entry) {
for _, e := range entries {
if e.Description != "" {
b.WriteString("/* " + strings.ReplaceAll(e.Description, "*/", "* /") + " */\n")
}
b.WriteString(stringsQuote(e.Key) + " = " + stringsQuote(e.Text) + ";\n")
}
- return b.Bytes()
}
var stringsLine = regexp.MustCompile(`(?s)"((?:[^"\\]|\\.)*)"\s*=\s*"((?:[^"\\]|\\.)*)"\s*;`)
@@ -561,8 +492,7 @@ func utf16ToUTF8(b []byte) []byte {
// --- Java .properties (UTF-8) ---
-func renderProperties(entries []Entry) []byte {
- var b bytes.Buffer
+func renderProperties(b *bufio.Writer, entries []Entry) {
esc := strings.NewReplacer(`\`, `\\`, "\n", `\n`, "\t", `\t`, "=", `\=`, ":", `\:`)
for _, e := range entries {
if e.Description != "" {
@@ -570,7 +500,6 @@ func renderProperties(entries []Entry) []byte {
}
b.WriteString(esc.Replace(e.Key) + "=" + strings.NewReplacer(`\`, `\\`, "\n", `\n`, "\t", `\t`).Replace(e.Text) + "\n")
}
- return b.Bytes()
}
func parseProperties(data []byte) (map[string]string, error) {
diff --git a/internal/formats/formats_test.go b/internal/formats/formats_test.go
index b806fd3..e45efb5 100644
--- a/internal/formats/formats_test.go
+++ b/internal/formats/formats_test.go
@@ -49,11 +49,3 @@ func TestCSV(t *testing.T) {
}
}
-func TestNestConflict(t *testing.T) {
- out := nest([]Entry{{Key: "a", Text: "A"}, {Key: "a.b", Text: "AB"}})
- flat := map[string]string{}
- flatten("", out, flat)
- if flat["a"] != "A" || flat["a.b"] != "AB" {
- t.Fatalf("conflict lost data: %v", flat)
- }
-}
diff --git a/internal/formats/write.go b/internal/formats/write.go
new file mode 100644
index 0000000..53c156a
--- /dev/null
+++ b/internal/formats/write.go
@@ -0,0 +1,204 @@
+package formats
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "io"
+ "sort"
+ "strings"
+ "unicode/utf8"
+)
+
+// Render returns a single-language file. Write streams it instead.
+func Render(format string, entries []Entry) ([]byte, error) {
+ var buf bytes.Buffer
+ err := Write(&buf, format, entries)
+ return buf.Bytes(), err
+}
+
+// Write streams a single-language file to w without building it in memory first.
+// entries is sorted in place.
+func Write(w io.Writer, format string, entries []Entry) error {
+ bw := bufio.NewWriterSize(w, 64<<10)
+ switch format {
+ case "json":
+ sortByKey(entries)
+ writeFlatJSON(bw, entries)
+ case "json-nested":
+ writeNestedJSON(bw, entries)
+ case "po":
+ sortByKey(entries)
+ renderPO(bw, entries)
+ case "android":
+ sortByKey(entries)
+ renderAndroid(bw, entries)
+ case "ios":
+ sortByKey(entries)
+ renderStrings(bw, entries)
+ case "properties":
+ sortByKey(entries)
+ renderProperties(bw, entries)
+ default:
+ return fmt.Errorf("unsupported format %q", format)
+ }
+ return bw.Flush()
+}
+
+func sortByKey(entries []Entry) {
+ sort.Slice(entries, func(i, j int) bool { return entries[i].Key < entries[j].Key })
+}
+
+// writeJSONString writes s as a JSON string, escaped like encoding/json without HTML escaping.
+func writeJSONString(w *bufio.Writer, s string) {
+ const hex = "0123456789abcdef"
+ w.WriteByte('"')
+ start := 0
+ for i := 0; i < len(s); {
+ c := s[i]
+ if c < utf8.RuneSelf {
+ if c >= 0x20 && c != '"' && c != '\\' {
+ i++
+ continue
+ }
+ w.WriteString(s[start:i])
+ switch c {
+ case '"', '\\':
+ w.WriteByte('\\')
+ w.WriteByte(c)
+ case '\n':
+ w.WriteString(`\n`)
+ case '\r':
+ w.WriteString(`\r`)
+ case '\t':
+ w.WriteString(`\t`)
+ default:
+ w.WriteString(`\u00`)
+ w.WriteByte(hex[c>>4])
+ w.WriteByte(hex[c&0xf])
+ }
+ i++
+ start = i
+ continue
+ }
+ r, size := utf8.DecodeRuneInString(s[i:])
+ switch {
+ case r == utf8.RuneError && size == 1:
+ w.WriteString(s[start:i])
+ w.WriteString(`\u` + "fffd") // the JSON escape for U+FFFD, as encoding/json writes it
+ case r == '
' || r == '
':
+ w.WriteString(s[start:i])
+ w.WriteString(`\u202`)
+ w.WriteByte(hex[r&0xf])
+ default:
+ i += size
+ continue
+ }
+ i += size
+ start = i
+ }
+ w.WriteString(s[start:])
+ w.WriteByte('"')
+}
+
+// writeFlatJSON writes {"key": "text", …}, indented with two spaces like json.MarshalIndent.
+func writeFlatJSON(w *bufio.Writer, entries []Entry) {
+ if len(entries) == 0 {
+ w.WriteString("{}\n")
+ return
+ }
+ w.WriteString("{\n")
+ for i, e := range entries {
+ w.WriteString(" ")
+ writeJSONString(w, e.Key)
+ w.WriteString(": ")
+ writeJSONString(w, e.Text)
+ if i < len(entries)-1 {
+ w.WriteByte(',')
+ }
+ w.WriteByte('\n')
+ }
+ w.WriteString("}\n")
+}
+
+// nestedPath splits a key on dots. When a shorter key is itself a text ("a" and "a.b"),
+// the rest of the longer key stays joined at that level: {"a": "A", "a.b": "AB"}.
+func nestedPath(key string, isKey map[string]bool) []string {
+ parts := strings.Split(key, ".")
+ for i := 1; i < len(parts); i++ {
+ if isKey[strings.Join(parts[:i], ".")] {
+ return append(parts[:i-1:i-1], strings.Join(parts[i-1:], "."))
+ }
+ }
+ return parts
+}
+
+// writeNestedJSON streams {"a": {"b": "text"}}, walking the keys sorted by path segment.
+func writeNestedJSON(w *bufio.Writer, entries []Entry) {
+ if len(entries) == 0 {
+ w.WriteString("{}\n")
+ return
+ }
+ isKey := make(map[string]bool, len(entries))
+ for _, e := range entries {
+ isKey[e.Key] = true
+ }
+ type item struct {
+ path []string
+ text string
+ }
+ items := make([]item, len(entries))
+ for i, e := range entries {
+ items[i] = item{nestedPath(e.Key, isKey), e.Text}
+ }
+ sort.Slice(items, func(i, j int) bool {
+ a, b := items[i].path, items[j].path
+ for k := 0; k < len(a) && k < len(b); k++ {
+ if a[k] != b[k] {
+ return a[k] < b[k]
+ }
+ }
+ return len(a) < len(b)
+ })
+
+ indent := func(depth int) {
+ for range depth {
+ w.WriteString(" ")
+ }
+ }
+ w.WriteString("{\n")
+ var open []string // path of the objects currently open below the root
+ for n, it := range items {
+ parent := it.path[:len(it.path)-1]
+ common := 0
+ for common < len(open) && common < len(parent) && open[common] == parent[common] {
+ common++
+ }
+ for len(open) > common { // close the objects this key is not in
+ w.WriteByte('\n')
+ indent(len(open))
+ w.WriteByte('}')
+ open = open[:len(open)-1]
+ }
+ if n > 0 {
+ w.WriteString(",\n")
+ }
+ for _, seg := range parent[common:] { // open the objects it needs
+ indent(len(open) + 1)
+ writeJSONString(w, seg)
+ w.WriteString(": {\n")
+ open = append(open, seg)
+ }
+ indent(len(open) + 1)
+ writeJSONString(w, it.path[len(it.path)-1])
+ w.WriteString(": ")
+ writeJSONString(w, it.text)
+ }
+ for len(open) > 0 {
+ w.WriteByte('\n')
+ indent(len(open))
+ w.WriteByte('}')
+ open = open[:len(open)-1]
+ }
+ w.WriteString("\n}\n")
+}
diff --git a/internal/formats/write_test.go b/internal/formats/write_test.go
new file mode 100644
index 0000000..43f297e
--- /dev/null
+++ b/internal/formats/write_test.go
@@ -0,0 +1,61 @@
+package formats
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "testing"
+)
+
+func TestNestConflict(t *testing.T) {
+ data, err := Render("json-nested", []Entry{{Key: "a", Text: "A"}, {Key: "a.b", Text: "AB"}, {Key: "a.b.c", Text: "ABC"}, {Key: "x.y", Text: "XY"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ flat, err := Parse("json", data)
+ if err != nil {
+ t.Fatalf("%v\n%s", err, data)
+ }
+ if flat["a"] != "A" || flat["a.b"] != "AB" || flat["a.b.c"] != "ABC" || flat["x.y"] != "XY" || len(flat) != 4 {
+ t.Fatalf("conflict lost data: %v\n%s", flat, data)
+ }
+}
+
+// The streamed writers must produce exactly what encoding/json would.
+func TestJSONMatchesEncodingJSON(t *testing.T) {
+ texts := []string{"plain", `quote " and \ backslash`, "line\nbreak\ttab\r", "ctrl \x01\x1f", "&",
+ "emoji 🎉 ção 日本", "sep
", "bad \xff utf8", ""}
+ var entries []Entry
+ flatWant := map[string]string{}
+ nestWant := map[string]any{}
+ grp := map[string]any{}
+ for i, tx := range texts {
+ k := fmt.Sprintf("k%d", i)
+ entries = append(entries, Entry{Key: k, Text: tx}, Entry{Key: "grp." + k, Text: tx}, Entry{Key: "a.b.c." + k, Text: tx})
+ flatWant[k], flatWant["grp."+k], flatWant["a.b.c."+k] = tx, tx, tx
+ nestWant[k] = tx
+ grp[k] = tx
+ }
+ nestWant["grp"] = grp
+ nestWant["a"] = map[string]any{"b": map[string]any{"c": grp}}
+
+ enc := func(v any) string {
+ var b bytes.Buffer
+ e := json.NewEncoder(&b)
+ e.SetEscapeHTML(false)
+ e.SetIndent("", " ")
+ e.Encode(v)
+ return b.String()
+ }
+ got, _ := Render("json", append([]Entry(nil), entries...))
+ if want := enc(flatWant); string(got) != want {
+ t.Fatalf("flat JSON differs\ngot:\n%s\nwant:\n%s", got, want)
+ }
+ got, _ = Render("json-nested", append([]Entry(nil), entries...))
+ if want := enc(nestWant); string(got) != want {
+ t.Fatalf("nested JSON differs\ngot:\n%s\nwant:\n%s", got, want)
+ }
+ if got, _ := Render("json-nested", nil); string(got) != "{}\n" {
+ t.Fatalf("empty: %q", got)
+ }
+}
diff --git a/internal/server/io.go b/internal/server/io.go
index 53052f1..1359d66 100644
--- a/internal/server/io.go
+++ b/internal/server/io.go
@@ -387,7 +387,7 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx)
return err
}
}
- render := func(texts map[int64]string, namespace string) ([]byte, error) {
+ entriesFor := func(texts map[int64]string, namespace string) []formats.Entry {
entries := []formats.Entry{}
for _, k := range keys {
if k.ns != namespace {
@@ -402,7 +402,7 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx)
}
entries = append(entries, formats.Entry{Key: k.name, Text: text, Description: k.desc})
}
- return formats.Render(format, entries)
+ return entries
}
namespaces := keyNamespaces(keys)
@@ -415,14 +415,12 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx)
if err != nil {
return err
}
- body, err := render(texts, nsName)
- if err != nil {
- return err
- }
- return sendFile(w, exportName(info, codes[0], ""), contentType(info.Ext), body)
+ w.Header().Set("Content-Type", contentType(info.Ext))
+ w.Header().Set("Content-Disposition", `attachment; filename="`+path.Base(exportName(info, codes[0], ""))+`"`)
+ return formats.Write(w, format, entriesFor(texts, nsName))
}
- // the ZIP is written straight to the response
+ // the ZIP is written straight to the response, one file at a time
w.Header().Set("Content-Type", "application/zip")
w.Header().Set("Content-Disposition", `attachment; filename="`+base+"-"+format+`.zip"`)
zw := zip.NewWriter(w)
@@ -432,15 +430,11 @@ func (s *Server) export(w http.ResponseWriter, r *http.Request, pc *projectCtx)
return err
}
for _, n := range namespaces {
- body, err := render(texts, n)
- if err != nil {
- return err
- }
f, err := zw.Create(exportName(info, code, n))
if err != nil {
return err
}
- if _, err := f.Write(body); err != nil {
+ if err := formats.Write(f, format, entriesFor(texts, n)); err != nil {
return err
}
}
diff --git a/internal/server/jobs.go b/internal/server/jobs.go
index 4027b0e..c57e4ff 100644
--- a/internal/server/jobs.go
+++ b/internal/server/jobs.go
@@ -28,6 +28,7 @@ var jobHandlers = map[string]jobHandler{}
type jobRunner struct {
wake chan struct{}
+ ctx context.Context
stop context.CancelFunc
wg sync.WaitGroup
mu sync.Mutex
@@ -50,9 +51,10 @@ type jobRun struct {
func (s *Server) startJobs() {
ctx, cancel := context.WithCancel(context.Background())
- s.jobs = &jobRunner{wake: make(chan struct{}, 1), stop: cancel, running: map[int64]context.CancelFunc{}}
+ s.jobs = &jobRunner{wake: make(chan struct{}, 1), ctx: ctx, stop: cancel, running: map[int64]context.CancelFunc{}}
// jobs left running by a previous process resume from their saved state
s.db.Exec(`UPDATE jobs SET status = 'queued' WHERE status = 'running'`)
+ s.flushSearchLater() // keys from an upgrade or an interrupted flush
for range jobWorkers {
s.jobs.wg.Add(1)
go s.jobWorker(ctx)
@@ -151,6 +153,7 @@ func (s *Server) runJob(parent context.Context, j *jobRun) {
switch {
case err == nil:
s.finishJob(j.ID, "done", "", result)
+ s.flushSearchLater() // imports change many keys; index them now rather than on the next search
case errors.Is(err, errJobCancelled):
s.finishJob(j.ID, "cancelled", "", result)
case parent.Err() != nil:
diff --git a/internal/server/keys.go b/internal/server/keys.go
index c8d912c..ff4d56c 100644
--- a/internal/server/keys.go
+++ b/internal/server/keys.go
@@ -53,10 +53,23 @@ func (s *Server) listKeys(w http.ResponseWriter, r *http.Request, pc *projectCtx
args = append(args, ns[0])
}
if term := strings.TrimSpace(q.Get("q")); term != "" {
- like := likeArg(term)
- where = append(where, `(k.name LIKE ? ESCAPE '\' OR k.description LIKE ? ESCAPE '\'
- OR EXISTS (SELECT 1 FROM translations t WHERE t.key_id = k.id AND t.text LIKE ? ESCAPE '\'))`)
- args = append(args, like, like, like)
+ if fts := trigramQuery(term); fts != "" {
+ // the trigram index narrows the keys down (keys changed since the last index update
+ // are checked directly), then a case-insensitive substring test in every script keeps
+ // only real matches. The index catches up in the background.
+ s.flushSearchLater()
+ needle := strings.ToLower(term)
+ where = append(where, `k.id IN (SELECT rowid FROM search_index WHERE search_index MATCH ? UNION ALL SELECT key_id FROM search_dirty)
+ AND (lowlex_contains(k.name, ?) OR lowlex_contains(k.description, ?)
+ OR EXISTS (SELECT 1 FROM translations t WHERE t.key_id = k.id AND lowlex_contains(t.text, ?)))`)
+ args = append(args, fts, needle, needle, needle)
+ } else {
+ // one or two characters: too short for trigrams, scan with LIKE
+ like := likeArg(term)
+ where = append(where, `(k.name LIKE ? ESCAPE '\' OR k.description LIKE ? ESCAPE '\'
+ OR EXISTS (SELECT 1 FROM translations t WHERE t.key_id = k.id AND t.text LIKE ? ESCAPE '\'))`)
+ args = append(args, like, like, like)
+ }
}
if tag := strings.TrimSpace(q.Get("tag")); tag != "" {
where = append(where, "EXISTS (SELECT 1 FROM json_each(k.tags) WHERE value = ?)")
diff --git a/internal/server/load_test.go b/internal/server/load_test.go
index 6cff0e2..f8c77d3 100644
--- a/internal/server/load_test.go
+++ b/internal/server/load_test.go
@@ -52,8 +52,27 @@ func TestTolgeeImportLoad(t *testing.T) {
t.Logf("%d keys imported in %s (%.0f keys/s); slowest key list during import %s; heap in use %d MB",
n, took.Round(time.Millisecond), float64(n)/took.Seconds(), worst.Round(time.Millisecond), mem.HeapInuse>>20)
- // a page of the editor, a search and the project stats on the full project
- for _, path := range []string{"/api/projects/big-app/keys?per=50&page=1500", "/api/projects/big-app/keys?q=Texto+15999", "/api/projects/big-app", "/api/projects"} {
+ // the first search waits for the index of everything just imported
+ t0 := time.Now()
+ admin.do("GET", "/api/projects/big-app/keys?q=Texto+15999", nil, 200)
+ t.Logf("first search (indexes %d keys): %s", n, time.Since(t0).Round(time.Millisecond))
+
+ // searches stay correct while the index catches up in the background; then they are fast
+ t0 = time.Now()
+ for time.Since(t0) < time.Minute {
+ t1 := time.Now()
+ admin.do("GET", "/api/projects/big-app/keys?q=xto+1234", nil, 200)
+ if time.Since(t1) < 150*time.Millisecond {
+ break
+ }
+ time.Sleep(time.Second)
+ }
+ time.Sleep(10 * time.Second) // let the last index batches and the WAL checkpoint finish
+ t.Logf("index caught up after %s", time.Since(t0).Round(time.Second))
+
+ // a page of the editor, searches and the project stats on the full project
+ for _, path := range []string{"/api/projects/big-app/keys?per=50&page=1500", "/api/projects/big-app/keys?q=Texto+15999",
+ "/api/projects/big-app/keys?q=xto+1234", "/api/projects/big-app/keys?q=ctx", "/api/projects/big-app/keys?q=7", "/api/projects/big-app", "/api/projects"} {
t0 := time.Now()
admin.do("GET", path, nil, 200)
t.Logf("GET %s: %s", path, time.Since(t0).Round(time.Millisecond))
diff --git a/internal/server/search.go b/internal/server/search.go
new file mode 100644
index 0000000..6e3a8f1
--- /dev/null
+++ b/internal/server/search.go
@@ -0,0 +1,104 @@
+package server
+
+import (
+ "context"
+ "log"
+ "strings"
+ "sync"
+ "sync/atomic"
+)
+
+// Search uses an FTS5 trigram index with one row per key (see store/schema3.sql). Writes
+// mark keys dirty; flushSearch folds them in the background, so the index never slows down
+// writing, and searches check dirty keys directly until then.
+
+var searchMu sync.Mutex
+
+const searchBatch = 1000
+
+// flushSearch indexes every key marked dirty since the last flush.
+func (s *Server) flushSearch(ctx context.Context) error {
+ searchMu.Lock()
+ defer searchMu.Unlock()
+ for {
+ var pending int
+ if err := s.db.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM search_dirty)`).Scan(&pending); err != nil || pending == 0 {
+ return err
+ }
+ tx, err := s.db.BeginTx(ctx, nil)
+ if err != nil {
+ return err
+ }
+ rows, err := tx.QueryContext(ctx, `SELECT key_id FROM search_dirty LIMIT ?`, searchBatch)
+ if err != nil {
+ tx.Rollback()
+ return err
+ }
+ var ids []int64
+ for rows.Next() {
+ var id int64
+ rows.Scan(&id)
+ ids = append(ids, id)
+ }
+ rows.Close()
+ for _, id := range ids {
+ if _, err := tx.ExecContext(ctx, `DELETE FROM search_index WHERE rowid = ?`, id); err != nil {
+ tx.Rollback()
+ return err
+ }
+ if _, err := tx.ExecContext(ctx, `INSERT INTO search_index (rowid, text)
+ SELECT k.id, k.name || char(10) || k.description || char(10) ||
+ COALESCE((SELECT group_concat(t.text, char(10)) FROM translations t WHERE t.key_id = k.id), '')
+ FROM keys k WHERE k.id = ?`, id); err != nil {
+ tx.Rollback()
+ return err
+ }
+ if _, err := tx.ExecContext(ctx, `DELETE FROM search_dirty WHERE key_id = ?`, id); err != nil {
+ tx.Rollback()
+ return err
+ }
+ }
+ if err := tx.Commit(); err != nil {
+ return err
+ }
+ }
+}
+
+var flushing atomic.Bool
+
+// flushSearchLater indexes dirty keys in the background, e.g. after an edit or a large import.
+// One flush runs at a time; Close waits for it.
+func (s *Server) flushSearchLater() {
+ if !flushing.CompareAndSwap(false, true) {
+ return
+ }
+ s.jobs.wg.Add(1)
+ go func() {
+ defer s.jobs.wg.Done()
+ defer flushing.Store(false)
+ if err := s.flushSearch(s.jobs.ctx); err != nil && s.jobs.ctx.Err() == nil {
+ log.Printf("search index: %v", err)
+ }
+ }()
+}
+
+// trigramQuery turns a search term into an FTS5 query that matches every text containing it:
+// all of its trigrams must be present. Terms shorter than three characters cannot use the
+// index and return "".
+func trigramQuery(term string) string {
+ r := []rune(strings.ToLower(term))
+ if len(r) < 3 {
+ return ""
+ }
+ seen := map[string]bool{}
+ var parts []string
+ for i := 0; i+3 <= len(r); i++ {
+ g := string(r[i : i+3])
+ if seen[g] {
+ continue
+ }
+ seen[g] = true
+ parts = append(parts, `"`+strings.ReplaceAll(g, `"`, `""`)+`"`)
+ }
+ return strings.Join(parts, " AND ")
+}
diff --git a/internal/server/search_test.go b/internal/server/search_test.go
new file mode 100644
index 0000000..3793cb9
--- /dev/null
+++ b/internal/server/search_test.go
@@ -0,0 +1,61 @@
+package server
+
+import (
+ "fmt"
+ "net/url"
+ "testing"
+)
+
+func TestSearch(t *testing.T) {
+ _, newClient := newTestServer(t)
+ admin := newClient()
+ admin.do("POST", "/api/auth/login", map[string]string{"username": "admin", "password": DefaultAdminPassword}, 200)
+ admin.do("POST", "/api/auth/password", map[string]string{"current_password": DefaultAdminPassword, "new_password": "search-test-password"}, 200)
+ admin.do("POST", "/api/projects", map[string]any{"name": "App", "languages": []string{"pt-BR"}}, 201)
+ admin.upload("/api/projects/app/import", "en.json", `{"menu.save":"Save changes","menu.quit":"Quit the game","ach.desc":""}`, nil, 200)
+ admin.upload("/api/projects/app/import", "pt-BR.json", `{"menu.save":"Salvar alterações","menu.quit":"Sair do jogo"}`, nil, 200)
+ k := admin.do("POST", "/api/projects/app/keys", map[string]any{"name": "dialog.title", "description": "Shown when the dragon appears"}, 201)
+ dialogID := int(k["key"].(map[string]any)["id"].(float64))
+
+ names := func(q string) map[string]bool {
+ t.Helper()
+ res := admin.do("GET", "/api/projects/app/keys?q="+url.QueryEscape(q), nil, 200)
+ out := map[string]bool{}
+ for _, k := range res["keys"].([]any) {
+ out[k.(map[string]any)["name"].(string)] = true
+ }
+ if int(res["total"].(float64)) != len(out) {
+ t.Fatalf("q=%q: total %v, keys %v", q, res["total"], out)
+ }
+ return out
+ }
+ expect := func(q string, want ...string) {
+ t.Helper()
+ got := names(q)
+ if len(got) != len(want) {
+ t.Fatalf("q=%q: got %v, want %v", q, got, want)
+ }
+ for _, w := range want {
+ if !got[w] {
+ t.Fatalf("q=%q: got %v, want %v", q, got, want)
+ }
+ }
+ }
+
+ expect("change", "menu.save") // inside a word
+ expect("ALTERAÇ", "menu.save") // accents, upper case (FTS) then LIKE confirms
+ expect("menu.qu", "menu.quit") // key name
+ expect("dragon", "dialog.title") // description
+ expect("jo", "menu.quit") // two letters: no index, plain LIKE
+ expect("e", "menu.save", "menu.quit", "dialog.title", "ach.desc") // one letter, key names count
+ expect("nothing like this")
+
+ // edits reach the index on the next search
+ admin.do("PUT", fmt.Sprintf("/api/projects/app/translations/%d/en", dialogID), map[string]any{"text": "A wild dragon appears"}, 200)
+ expect("wild drag", "dialog.title")
+ admin.do("PUT", fmt.Sprintf("/api/projects/app/translations/%d/en", dialogID), map[string]any{"text": "A calm sea"}, 200)
+ expect("wild drag")
+ admin.do("DELETE", fmt.Sprintf("/api/projects/app/keys/%d", dialogID), nil, 200)
+ expect("dragon")
+ expect("calm sea")
+}
diff --git a/internal/store/schema3.sql b/internal/store/schema3.sql
new file mode 100644
index 0000000..fad9b15
--- /dev/null
+++ b/internal/store/schema3.sql
@@ -0,0 +1,32 @@
+-- LowLex schema, version 3: full-text search.
+-- One row per key (rowid = key id) with its name, description and every text, split into
+-- trigrams so any substring of 3+ characters can be found without scanning every text.
+-- Writes only mark the key in search_dirty (with WHERE NOT EXISTS: inside a trigger, OR IGNORE
+-- would take the outer statement's conflict policy and fail during upserts); the server folds dirty keys into the index in
+-- batches, so imports stay fast.
+
+CREATE VIRTUAL TABLE search_index USING fts5(text, content='', contentless_delete=1, tokenize='trigram', detail='none');
+
+CREATE TABLE search_dirty (key_id INTEGER PRIMARY KEY);
+
+CREATE TRIGGER search_translation_insert AFTER INSERT ON translations BEGIN
+ INSERT INTO search_dirty (key_id) SELECT new.key_id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = new.key_id);
+END;
+CREATE TRIGGER search_translation_update AFTER UPDATE OF text ON translations BEGIN
+ INSERT INTO search_dirty (key_id) SELECT new.key_id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = new.key_id);
+END;
+CREATE TRIGGER search_translation_delete AFTER DELETE ON translations BEGIN
+ INSERT INTO search_dirty (key_id) SELECT old.key_id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = old.key_id);
+END;
+CREATE TRIGGER search_key_insert AFTER INSERT ON keys BEGIN
+ INSERT INTO search_dirty (key_id) SELECT new.id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = new.id);
+END;
+CREATE TRIGGER search_key_update AFTER UPDATE OF name, description ON keys BEGIN
+ INSERT INTO search_dirty (key_id) SELECT new.id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = new.id);
+END;
+CREATE TRIGGER search_key_delete AFTER DELETE ON keys BEGIN
+ INSERT INTO search_dirty (key_id) SELECT old.id WHERE NOT EXISTS (SELECT 1 FROM search_dirty WHERE key_id = old.id);
+END;
+
+-- existing keys are indexed on the first start after the upgrade
+INSERT INTO search_dirty (key_id) SELECT id FROM keys;
diff --git a/internal/store/store.go b/internal/store/store.go
index 436ea6d..7a0bbba 100644
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -3,22 +3,43 @@ package store
import (
"database/sql"
+ "database/sql/driver"
_ "embed"
"fmt"
"os"
"path/filepath"
+ "strings"
- _ "modernc.org/sqlite"
+ "modernc.org/sqlite"
)
+// lowlex_contains(text, needle) is a substring test that ignores case in every script,
+// unlike LIKE, which folds ASCII only. needle must already be lower case.
+func init() {
+ sqlite.MustRegisterDeterministicScalarFunction("lowlex_contains", 2, func(_ *sqlite.FunctionContext, args []driver.Value) (driver.Value, error) {
+ text, _ := args[0].(string)
+ needle, _ := args[1].(string)
+ if b, ok := args[0].([]byte); ok {
+ text = string(b)
+ }
+ if strings.Contains(strings.ToLower(text), needle) {
+ return int64(1), nil
+ }
+ return int64(0), nil
+ })
+}
+
//go:embed schema.sql
var schemaV1 string
//go:embed schema2.sql
var schemaV2 string
+//go:embed schema3.sql
+var schemaV3 string
+
// migrations[i] moves the database from user_version i to i+1.
-var migrations = []string{schemaV1, schemaV2}
+var migrations = []string{schemaV1, schemaV2, schemaV3}
// Open creates the data directory if needed, opens lowlex.db and migrates it.
func Open(dataDir string) (*sql.DB, error) {