Memory limit from the container, streamed exports, trigram search index

cmd/lowlex/main.go

@@ -84,6 +84,7 @@ func main() {
 }
 
 func serve(cfg config.Config) error {
+	setMemoryLimit()
 	db, err := store.Open(cfg.DataDir)
 	if err != nil {
 		return err

cmd/lowlex/memlimit.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"log"
+	"os"
+	"runtime/debug"
+	"strconv"
+	"strings"
+)
+
+// setMemoryLimit keeps the Go heap under the container's memory limit. Without it the
+// garbage collector does not know about the cgroup limit, lets the heap grow to it and the
+// kernel then throttles the process. GOMEMLIMIT, when set, wins.
+func setMemoryLimit() {
+	if os.Getenv("GOMEMLIMIT") != "" {
+		return
+	}
+	for _, path := range []string{
+		"/sys/fs/cgroup/memory.max",                   // cgroup v2
+		"/sys/fs/cgroup/memory/memory.limit_in_bytes", // cgroup v1
+	} {
+		b, err := os.ReadFile(path)
+		if err != nil {
+			continue
+		}
+		limit, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
+		if err != nil || limit <= 0 || limit > 1<<50 { // "max" or the v1 "unlimited" value
+			return
+		}
+		soft := limit / 10 * 8
+		debug.SetMemoryLimit(soft)
+		log.Printf("memory limit %d MB: keeping the heap under %d MB", limit>>20, soft>>20)
+		return
+	}
+}

internal/formats/formats.go

@@ -90,39 +90,6 @@ func Parse(format string, data []byte) (map[string]string, error) {
 	return nil, fmt.Errorf("unsupported format %q", format)
 }
 
-// Render writes a single-language file.
-func Render(format string, entries []Entry) ([]byte, error) {
-	sort.Slice(entries, func(i, j int) bool { return entries[i].Key < entries[j].Key })
-	switch format {
-	case "json":
-		m := make(map[string]string, len(entries))
-		for _, e := range entries {
-			m[e.Key] = e.Text
-		}
-		return marshal(m)
-	case "json-nested":
-		return marshal(nest(entries))
-	case "po":
-		return renderPO(entries), nil
-	case "android":
-		return renderAndroid(entries), nil
-	case "ios":
-		return renderStrings(entries), nil
-	case "properties":
-		return renderProperties(entries), nil
-	}
-	return nil, fmt.Errorf("unsupported format %q", format)
-}
-
-func marshal(v any) ([]byte, error) {
-	var buf bytes.Buffer
-	enc := json.NewEncoder(&buf)
-	enc.SetEscapeHTML(false)
-	enc.SetIndent("", "  ")
-	err := enc.Encode(v)
-	return buf.Bytes(), err
-}
-
 // --- JSON ---
 
 func parseJSON(data []byte) (map[string]string, error) {
@@ -166,36 +133,6 @@ func flatten(prefix string, v any, out map[string]string) {
 	}
 }
 
-func nest(entries []Entry) map[string]any {
-	root := map[string]any{}
-	for _, e := range entries {
-		parts := strings.Split(e.Key, ".")
-		node := root
-		for i, p := range parts {
-			if i == len(parts)-1 {
-				if _, isObj := node[p].(map[string]any); isObj {
-					// "a" and "a.b" both exist: keep "a" next to its children with the full key
-					root[e.Key] = e.Text
-				} else {
-					node[p] = e.Text
-				}
-				break
-			}
-			next, ok := node[p].(map[string]any)
-			if !ok {
-				if _, isLeaf := node[p]; isLeaf {
-					node[strings.Join(parts[i:], ".")] = e.Text
-					break
-				}
-				next = map[string]any{}
-				node[p] = next
-			}
-			node = next
-		}
-	}
-	return root
-}
-
 // --- CSV: key, description, then one column per language ---
 
 // ParseCSV returns texts by language code then key.
@@ -325,8 +262,7 @@ func poUnquote(s string) string {
 	return b.String()
 }
 
-func renderPO(entries []Entry) []byte {
-	var b bytes.Buffer
+func renderPO(b *bufio.Writer, entries []Entry) {
 	b.WriteString("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n\n")
 	for _, e := range entries {
 		if e.Description != "" {
@@ -336,7 +272,6 @@ func renderPO(entries []Entry) []byte {
 		}
 		b.WriteString("msgid " + poQuote(e.Key) + "\nmsgstr " + poQuote(e.Text) + "\n\n")
 	}
-	return b.Bytes()
 }
 
 func parsePO(data []byte) (map[string]string, error) {
@@ -460,8 +395,7 @@ func androidUnescape(s string) string {
 	return b.String()
 }
 
-func renderAndroid(entries []Entry) []byte {
-	var b bytes.Buffer
+func renderAndroid(b *bufio.Writer, entries []Entry) {
 	b.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<resources>\n")
 	for _, e := range entries {
 		name := strings.NewReplacer(".", "_", "-", "_", " ", "_").Replace(e.Key)
@@ -471,7 +405,6 @@ func renderAndroid(entries []Entry) []byte {
 		b.WriteString(`    <string name="` + name + `">` + androidEscape(e.Text) + "</string>\n")
 	}
 	b.WriteString("</resources>\n")
-	return b.Bytes()
 }
 
 func parseAndroid(data []byte) (map[string]string, error) {
@@ -507,15 +440,13 @@ func stringsQuote(s string) string {
 	return `"` + r.Replace(s) + `"`
 }
 
-func renderStrings(entries []Entry) []byte {
-	var b bytes.Buffer
+func renderStrings(b *bufio.Writer, entries []Entry) {
 	for _, e := range entries {
 		if e.Description != "" {
 			b.WriteString("/* " + strings.ReplaceAll(e.Description, "*/", "* /") + " */\n")
 		}
 		b.WriteString(stringsQuote(e.Key) + " = " + stringsQuote(e.Text) + ";\n")
 	}
-	return b.Bytes()
 }
 
 var stringsLine = regexp.MustCompile(`(?s)"((?:[^"\\]|\\.)*)"\s*=\s*"((?:[^"\\]|\\.)*)"\s*;`)
@@ -561,16 +492,14 @@ func utf16ToUTF8(b []byte) []byte {
 
 // --- Java .properties (UTF-8) ---
 
-func renderProperties(entries []Entry) []byte {
-	var b bytes.Buffer
+func renderProperties(b *bufio.Writer, entries []Entry) {
 	esc := strings.NewReplacer(`\`, `\\`, "\n", `\n`, "\t", `\t`, "=", `\=`, ":", `\:`)
 	for _, e := range entries {
 		if e.Description != "" {
 			b.WriteString("# " + strings.ReplaceAll(e.Description, "\n", " ") + "\n")
 		}
 		b.WriteString(esc.Replace(e.Key) + "=" + strings.NewReplacer(`\`, `\\`, "\n", `\n`, "\t", `\t`).Replace(e.Text) + "\n")
 	}
-	return b.Bytes()
 }
 
 func parseProperties(data []byte) (map[string]string, error) {

internal/formats/formats_test.go

@@ -49,11 +49,3 @@ func TestCSV(t *testing.T) {
 	}
 }
 
-func TestNestConflict(t *testing.T) {
-	out := nest([]Entry{{Key: "a", Text: "A"}, {Key: "a.b", Text: "AB"}})
-	flat := map[string]string{}
-	flatten("", out, flat)
-	if flat["a"] != "A" || flat["a.b"] != "AB" {
-		t.Fatalf("conflict lost data: %v", flat)
-	}
-}