fix(streaming): keep the Go memory limit above the live heap so GC cannot spin

cmd/altmount/cmd/serve.go

@@ -139,6 +139,9 @@ func runServe(cmd *cobra.Command, args []string) error {
 		defer initialCache.Stop()
 	}
 	applySoftMemoryLimit(ctx, cfg)
+	// Keep the memory tier from pushing the live heap over the soft limit:
+	// under GC pressure the governor shrinks it, then restores it when calm.
+	go cacheSource.RunPressureGovernor(ctx)
 
 	// Background PAR2 repair: repairs missing articles and serves the patched
 	// payloads on the read path's hole branch.

config.sample.yaml

@@ -207,8 +207,11 @@ health:
 
 # WebDAV mount path configuration
 # Go soft memory limit in MB. Unset or 0: derived from segment_cache.memory_mb plus the PAR2 solver
-# budget plus 256 MB headroom (about 770 MB with defaults). Positive: use exactly this value.
-# -1: leave the Go runtime default. GOMEMLIMIT in the environment always wins.
+# budget plus headroom for three read-ahead windows, the provider connections' buffers and a 128 MB
+# base (about 930 MB with defaults and one 20-connection provider). Positive: use exactly this value.
+# -1: leave the Go runtime default. GOMEMLIMIT in the environment always wins. When the live heap
+# still presses on the limit, the segment cache memory tier is shrunk automatically so the collector
+# does not spin, and restored once the heap is calm.
 memory_limit_mb: 0
 mount_path: '' # WebDAV mount path, Example: '/mnt/remotes/altmount' or '/mnt/unionfs'. Must be an absolute path.
                # Windows examples: 'Z:' (drive letter via WinFsp) or 'C:\altmount' (directory mount)

docs/docs/3. Configuration/streaming.md

@@ -127,9 +127,9 @@ Reads check memory first, then disk, then fetch from the provider:
 
 Disk eviction runs automatically every 5 minutes, removing expired entries and enforcing the size limit via LRU (least recently used). Files that are currently open are never evicted. The memory tier evicts the least recently used articles as soon as it reaches `memory_mb`.
 
-**RAM usage:** while streaming, AltMount uses roughly `memory_mb` plus 300 MB. The extra is read-ahead buffers, connections and the Go runtime. Lower `memory_mb` on small boxes (128 MB is a good balance).
+**RAM usage:** while streaming, AltMount uses roughly `memory_mb` plus 300-450 MB, bounded by the Go memory limit described below. The extra is read-ahead buffers (up to 96 MB per open reader), about 256 KB per provider connection, and the Go runtime, so it grows with many-provider setups. Lower `memory_mb` on small boxes (128 MB is a good balance).
 
-**Go memory limit:** AltMount sets a Go soft memory limit so evicted articles are collected promptly instead of letting the heap double before each collection. The top-level `memory_limit_mb` controls it: unset or `0` derives the limit from `segment_cache.memory_mb` plus the PAR2 solver budget (`par2_repair.max_memory_mb` × `max_concurrent_jobs`) plus 256 MB headroom, about 770 MB with defaults; a positive value pins the limit (for example `512` on a box where only one stream and no repairs run); `-1` leaves the Go default. `GOMEMLIMIT` in the environment always takes precedence, so Docker users can also set it there. Keep the limit above what the process actually holds live: if live memory exceeds a soft limit the collector runs continuously and costs CPU without freeing anything.
+**Go memory limit:** AltMount sets a Go soft memory limit so evicted articles are collected promptly instead of letting the heap double before each collection. The top-level `memory_limit_mb` controls it: unset or `0` derives the limit from everything that holds live heap: `segment_cache.memory_mb`, the PAR2 solver budget (`par2_repair.max_memory_mb` × `max_concurrent_jobs`) when repair is on, three read-ahead windows (96 MB each), 256 KB per configured provider connection, and a 128 MB base. That is about 930 MB with defaults and one 20-connection provider, and it grows with large multi-provider setups because their connection buffers are live too. A positive value pins the limit (for example `512` on a box where only one stream and no repairs run); `-1` leaves the Go default. `GOMEMLIMIT` in the environment always takes precedence, so Docker users can also set it there. Keep the limit above what the process actually holds live: if live memory exceeds a soft limit the collector runs continuously and costs CPU without freeing anything. As a safety net, AltMount watches the runtime's GC CPU limiter and, when the live heap presses on the limit, shrinks the segment cache memory tier in steps (down to 32 MB) so read-ahead buffers fit under it, logging a warning; the tier is restored once the heap has been calm for a minute. Seeing that warning means `memory_limit_mb` is too low for your stream count and connection total, or `segment_cache.memory_mb` is too high.
 
 ### Tips
 

internal/config/manager.go

@@ -237,17 +237,49 @@ func (c SegmentCacheConfig) MemoryBytes() int64 {
 	return int64(max(*c.MemoryMB, 0)) << 20
 }
 
-// softMemoryHeadroomMB is what the process needs above the configured
-// budgets: read-ahead windows, connection buffers, metadata, and the runtime.
-const softMemoryHeadroomMB = 256
+// StreamReadAheadBytesCap bounds one reader's read-ahead window in bytes.
+// The usenet reader enforces it; it lives here so the soft memory limit can
+// budget for the windows that are live while files stream.
+const StreamReadAheadBytesCap int64 = 96 << 20
+
+// Soft memory limit headroom: what the process holds live above the memory
+// tier and the PAR2 solver. Every term scales with what actually allocates
+// so the limit stays above the live set; a limit below it makes the
+// collector run back to back and burn CPU without freeing anything.
+const (
+	// softMemoryBaseMB covers the runtime, metadata, HTTP and pool bookkeeping.
+	softMemoryBaseMB = 128
+	// softMemoryStreams is how many full read-ahead windows are budgeted:
+	// a mount typically keeps two chunk readers open plus one being torn down.
+	softMemoryStreams = 3
+	// softMemoryPerConnectionBytes is the read and write buffers plus TLS
+	// record state each pool connection pins while open.
+	softMemoryPerConnectionBytes int64 = 256 << 10
+)
+
+// softMemoryHeadroom is the headroom for this config's read-ahead and
+// connection footprint.
+func (c *Config) softMemoryHeadroom() int64 {
+	conns := int64(0)
+	for _, p := range c.Providers {
+		if p.Enabled != nil && !*p.Enabled {
+			continue
+		}
+		conns += int64(max(p.MaxConnections, 0))
+	}
+	return int64(softMemoryBaseMB)<<20 +
+		softMemoryStreams*StreamReadAheadBytesCap +
+		conns*softMemoryPerConnectionBytes
+}
 
 // SoftMemoryLimit is the Go soft memory limit to apply, or 0 to leave the
 // runtime alone. Without a limit the collector lets the heap reach twice the
 // live set, so a 256 MB memory tier costs 600+ MB of RSS. The automatic value
 // adds every budget that holds live heap (memory tier, PAR2 solver per
-// concurrent job) plus headroom, so the limit stays above the live set and the
-// collector never has to run back to back. A soft limit is only useful while
-// the memory tier is on: with it off the heap is small and bursty.
+// concurrent job, read-ahead windows, connection buffers) plus a base, so
+// the limit stays above the live set and the collector never has to run back
+// to back. A soft limit is only useful while the memory tier is on: with it
+// off the heap is small and bursty.
 func (c *Config) SoftMemoryLimit(gomemlimit string) int64 {
 	if gomemlimit != "" {
 		return 0
@@ -268,7 +300,7 @@ func (c *Config) SoftMemoryLimit(gomemlimit string) int64 {
 	if c.Par2Repair.Enabled != nil && *c.Par2Repair.Enabled {
 		par2 = int64(max(c.Par2Repair.MaxMemoryMB, 0)) * int64(max(c.Par2Repair.MaxConcurrentJobs, 1))
 	}
-	return cache + (par2+softMemoryHeadroomMB)<<20
+	return cache + par2<<20 + c.softMemoryHeadroom()
 }
 
 // WebDAVConfig represents WebDAV server configuration

internal/config/soft_memory_limit_test.go

@@ -9,14 +9,63 @@ func memLimitConfig(cacheMB int, par2MB, par2Jobs int) *Config {
 	c.Par2Repair.MaxConcurrentJobs = par2Jobs
 	enabled := true
 	c.Par2Repair.Enabled = &enabled
+	c.Providers = nil
 	return c
 }
 
+// headroomBytes is the expected headroom for a config with no providers:
+// runtime base plus the budgeted read-ahead windows.
+func headroomBytes() int64 {
+	return int64(softMemoryBaseMB)<<20 + softMemoryStreams*StreamReadAheadBytesCap
+}
+
+func withProviders(c *Config, conns ...int) *Config {
+	on := true
+	for _, n := range conns {
+		c.Providers = append(c.Providers, ProviderConfig{Host: "h", Port: 563, MaxConnections: n, Enabled: &on})
+	}
+	return c
+}
+
+func TestSoftMemoryLimitHeadroomScalesWithReadAheadAndConnections(t *testing.T) {
+	c := memLimitConfig(256, 256, 1)
+	disabled := false
+	c.Par2Repair.Enabled = &disabled
+	base := c.SoftMemoryLimit("")
+	if want := int64(256)<<20 + headroomBytes(); base != want {
+		t.Fatalf("SoftMemoryLimit without providers = %d, want %d", base, want)
+	}
+	// The user config that triggered a GC spiral: 341 connections across
+	// six providers and a 256 MB memory tier had a 512 MiB limit.
+	withProviders(c, 48, 80, 55, 55, 48, 55)
+	got := c.SoftMemoryLimit("")
+	if want := base + 341*softMemoryPerConnectionBytes; got != want {
+		t.Fatalf("SoftMemoryLimit with 341 connections = %d, want %d", got, want)
+	}
+	if got <= int64(700)<<20 {
+		t.Fatalf("SoftMemoryLimit with 341 connections = %d MB, must exceed the ~700 MB live set", got>>20)
+	}
+}
+
+func TestSoftMemoryLimitCountsBackupAndSkipsDisabledProviders(t *testing.T) {
+	c := memLimitConfig(256, 256, 1)
+	base := c.SoftMemoryLimit("")
+	on, off := true, false
+	c.Providers = []ProviderConfig{
+		{Host: "a", Port: 563, MaxConnections: 10, Enabled: &on},
+		{Host: "b", Port: 563, MaxConnections: 20, Enabled: &on, IsBackupProvider: &on},
+		{Host: "c", Port: 563, MaxConnections: 99, Enabled: &off},
+	}
+	if got, want := c.SoftMemoryLimit(""), base+30*softMemoryPerConnectionBytes; got != want {
+		t.Fatalf("SoftMemoryLimit = %d, want %d (backup counted, disabled skipped)", got, want)
+	}
+}
+
 func TestSoftMemoryLimitIgnoresPar2BudgetWhenRepairDisabled(t *testing.T) {
 	c := memLimitConfig(256, 256, 1)
 	disabled := false
 	c.Par2Repair.Enabled = &disabled
-	want := int64(256+softMemoryHeadroomMB) << 20
+	want := int64(256)<<20 + headroomBytes()
 	if got := c.SoftMemoryLimit(""); got != want {
 		t.Fatalf("SoftMemoryLimit with repair disabled = %d, want %d", got, want)
 	}
@@ -28,15 +77,15 @@ func TestSoftMemoryLimitIgnoresPar2BudgetWhenRepairDisabled(t *testing.T) {
 
 func TestSoftMemoryLimitAutoAddsCachePar2AndHeadroom(t *testing.T) {
 	c := memLimitConfig(256, 256, 1)
-	want := int64(256+256+softMemoryHeadroomMB) << 20
+	want := int64(256+256)<<20 + headroomBytes()
 	if got := c.SoftMemoryLimit(""); got != want {
 		t.Fatalf("SoftMemoryLimit = %d, want %d", got, want)
 	}
 }
 
 func TestSoftMemoryLimitAutoScalesPar2ByConcurrentJobs(t *testing.T) {
 	c := memLimitConfig(128, 100, 3)
-	want := int64(128+300+softMemoryHeadroomMB) << 20
+	want := int64(128+300)<<20 + headroomBytes()
 	if got := c.SoftMemoryLimit(""); got != want {
 		t.Fatalf("SoftMemoryLimit = %d, want %d", got, want)
 	}

internal/nzbfilesystem/segcache/pressure.go

@@ -0,0 +1,176 @@
+package segcache
+
+import (
+	"context"
+	"log/slog"
+	"math"
+	"runtime/metrics"
+	"time"
+)
+
+// The memory tier is the one large, elastic block of live heap. When the Go
+// soft memory limit sits below what the process holds live, the collector
+// runs back to back and the GC CPU limiter engages: CPU climbs to half the
+// machine and stays there for as long as the stream runs, while no memory is
+// freed because it is all live. The pressure governor watches for that state
+// and shrinks the tier so read-ahead windows and connection buffers fit
+// under the limit again, then restores it once the heap has been calm for a
+// while. RAM stays bounded by the limit; CPU never spirals.
+const (
+	pressureInterval = 2 * time.Second
+	// pressureHighFraction of the limit held live is treated as pressure
+	// before the limiter has to engage.
+	pressureHighFraction = 0.90
+	// pressureLowFraction of the limit is calm enough to start recovering.
+	pressureLowFraction = 0.70
+	// pressureRecoverAfter is how long the heap must stay calm per step back up.
+	pressureRecoverAfter = 60 * time.Second
+	// pressureSteps is how many steps take the tier from full to the floor.
+	pressureSteps = 4
+	// pressureFloor keeps a small tier for warm opens and fan-out sharing.
+	pressureFloor = int64(32) << 20
+)
+
+// noCeiling means the configured capacity applies unchanged.
+const noCeiling int64 = -1
+
+type pressureSample struct {
+	limiterCycle uint64 // /gc/limiter/last-enabled:gc-cycle
+	live         int64  // /gc/heap/live:bytes (marked heap of the last cycle; 0 before the first)
+	limit        int64  // /gc/gomemlimit:bytes
+}
+
+type pressureGovernor struct {
+	full      func() int64 // configured memory tier capacity
+	ceiling   int64
+	lastCycle uint64
+	baselined bool
+	calmSince time.Time
+}
+
+func newPressureGovernor(full func() int64) *pressureGovernor {
+	return &pressureGovernor{full: full, ceiling: noCeiling}
+}
+
+// step folds one sample into the governor and returns the ceiling to apply
+// and whether it changed. Pure so the policy is testable without a runtime.
+func (g *pressureGovernor) step(s pressureSample, now time.Time) (int64, bool) {
+	engaged := g.baselined && s.limiterCycle != g.lastCycle
+	g.lastCycle, g.baselined = s.limiterCycle, true
+
+	full := g.full()
+	if s.limit <= 0 || s.limit == math.MaxInt64 || full <= pressureFloor {
+		return g.set(noCeiling)
+	}
+	step := full / pressureSteps
+	current := g.ceiling
+	if current == noCeiling || current > full {
+		current = full
+	}
+	switch {
+	case engaged || float64(s.live) > float64(s.limit)*pressureHighFraction:
+		g.calmSince = now // recovery is counted from the last shrink
+		return g.set(g.normalize(max(current-step, pressureFloor), full))
+	case float64(s.live) < float64(s.limit)*pressureLowFraction:
+		if current >= full {
+			return g.set(noCeiling)
+		}
+		if g.calmSince.IsZero() {
+			g.calmSince = now
+			return g.set(g.normalize(current, full))
+		}
+		if now.Sub(g.calmSince) < pressureRecoverAfter {
+			return g.set(g.normalize(current, full))
+		}
+		g.calmSince = now
+		return g.set(g.normalize(current+step, full))
+	default:
+		g.calmSince = time.Time{}
+		return g.set(g.normalize(current, full))
+	}
+}
+
+func (g *pressureGovernor) normalize(c, full int64) int64 {
+	if c >= full {
+		return noCeiling
+	}
+	return c
+}
+
+func (g *pressureGovernor) set(c int64) (int64, bool) {
+	changed := c != g.ceiling
+	g.ceiling = c
+	return c, changed
+}
+
+var pressureMetricNames = []string{
+	"/gc/limiter/last-enabled:gc-cycle",
+	"/gc/heap/live:bytes",
+	"/gc/gomemlimit:bytes",
+}
+
+func readPressureSample() (pressureSample, bool) {
+	samples := make([]metrics.Sample, len(pressureMetricNames))
+	for i, n := range pressureMetricNames {
+		samples[i].Name = n
+	}
+	metrics.Read(samples)
+	for _, s := range samples {
+		if s.Value.Kind() != metrics.KindUint64 {
+			return pressureSample{}, false
+		}
+	}
+	return pressureSample{
+		limiterCycle: samples[0].Value.Uint64(),
+		live:         clampInt64(samples[1].Value.Uint64()),
+		limit:        clampInt64(samples[2].Value.Uint64()),
+	}, true
+}
+
+func clampInt64(v uint64) int64 {
+	if v > math.MaxInt64 {
+		return math.MaxInt64
+	}
+	return int64(v)
+}
+
+// RunPressureGovernor shrinks the memory tier while the Go soft memory limit
+// is under pressure and restores it when the heap is calm. Blocks until ctx
+// is cancelled; run it in its own goroutine.
+func (s *Source) RunPressureGovernor(ctx context.Context) {
+	g := newPressureGovernor(func() int64 { return s.getCfg().SegmentCache.MemoryBytes() })
+	t := time.NewTicker(pressureInterval)
+	defer t.Stop()
+	warned := false
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case now := <-t.C:
+			sample, ok := readPressureSample()
+			if !ok {
+				slog.ErrorContext(ctx, "Runtime GC metrics unavailable; memory pressure governor disabled")
+				return
+			}
+			ceiling, changed := g.step(sample, now)
+			if !changed {
+				continue
+			}
+			s.SetMemoryCeiling(ceiling)
+			attrs := []any{
+				"live_mb", sample.live >> 20, "limit_mb", sample.limit >> 20,
+				"tier_mb", s.effectiveMemoryBytes(s.getCfg().SegmentCache.MemoryBytes()) >> 20,
+			}
+			switch {
+			case ceiling == noCeiling:
+				warned = false // a later episode is news again
+				slog.InfoContext(ctx, "Memory pressure eased; segment cache memory tier restored", attrs...)
+			case !warned:
+				warned = true
+				slog.WarnContext(ctx, "Live heap is pressing on the Go memory limit; shrinking the segment cache memory tier to keep GC CPU down. Raise memory_limit_mb or lower segment_cache.memory_mb to avoid this.", attrs...)
+			default:
+				slog.InfoContext(ctx, "Memory pressure persists; segment cache memory tier shrunk further", attrs...)
+			}
+		}
+	}
+}