From 0290ae5506fe0ceea6767f9a1809402c26474828 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:58:32 +0100 Subject: [PATCH 1/6] fix(proxies): retry a copy the cap reset could not start, every minute while uncapped A reset whose start failed left the copy marked as the cap's and stopped until the next reset or a Start by hand. ProxyTick now starts marked copies that are down while the proxy is uncapped. Mirrors GeiserX/CashPilot#630. --- CHANGELOG.md | 1 + docs/proxies.md | 4 +-- internal/services/proxies.go | 34 ++++++++++++++++++++++- internal/services/proxies_test.go | 45 +++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e9150d..0c80d04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A copy the cap reset could not start stayed stopped for a whole period.** It kept its "stopped by the cap" mark, and nothing tried again until the next reset or a Start by hand. The once-a-minute pass now starts such a copy while the proxy is uncapped, and leaves alone copies you stopped yourself. - **Two apps on one Docker host could lose each other's Activity sample.** A sweep removed another app's helper the moment it exited, before its owner had read the log. Another app's helper is now removed only after it has been finished, or stuck before starting, for longer than the 30-second helper limit. A sock_diag dump the kernel ends with an error code is now an error, and a reading in which every dump of one snapshot failed is marked partial. - **An untouched money4band `user-config.json` filled the import table with placeholder rows.** An app that is off and holds only template placeholders is now left out, as the `.env` reader already did. The import notice and guides now point proxies at the Proxies section of the Fleet page, and the guides explain the missing CashFactory Bitping row and the refusal when a saved value differs from the file. `make catalog-check` now also compares the import mapping with the web app's, so the two importers cannot drift apart unseen. diff --git a/docs/proxies.md b/docs/proxies.md index 625f4ca..9dcb518 100644 --- a/docs/proxies.md +++ b/docs/proxies.md @@ -164,8 +164,8 @@ proxy is stopped and you get a notice. Only the copies that were running are marked as stopped by the cap, so a copy you had already stopped stays stopped after the reset. While the proxy is capped, Start is disabled on its copies: reset the usage or raise the cap first. A reset starts every marked copy; one -whose start fails keeps its mark, the failure shows in its events, and the next -reset or a Start by hand tries again. +whose start fails keeps its mark, the failure shows in its events, and every +minute after that the app tries again while the proxy stays uncapped. ## Tailscale exit nodes diff --git a/internal/services/proxies.go b/internal/services/proxies.go index f74cce4..69126ac 100644 --- a/internal/services/proxies.go +++ b/internal/services/proxies.go @@ -688,7 +688,11 @@ func (m *Manager) ProxyTick(ctx context.Context) ([]CapNotice, error) { continue } if reset && next.CappedAt == "" { - _, _ = m.releaseCapped(ctx, id, d) // the next reset or a start by hand retries + _, _ = m.releaseCapped(ctx, id, d) + } else if next.CappedAt == "" { + // A reset whose start failed leaves the copy marked as the cap's and down; + // while the proxy stays uncapped, every pass tries again. + m.restartCapStoppedDown(ctx, p, id, d, mine) } if next.CappedAt == "" { continue @@ -717,6 +721,34 @@ func (m *Manager) ProxyTick(ctx context.Context) ([]CapNotice, error) { return crossedNow, nil } +// restartCapStoppedDown starts this proxy's copies that are still marked as stopped +// by the cap and are not running. A copy the owner stopped is never marked, so it +// stays stopped; one that is running is left alone. +func (m *Manager) restartCapStoppedDown(ctx context.Context, p runtime.ProxyProvider, id int64, d proxies.Definition, status []runtime.ProxiedCopy) { + records, err := m.store.ListProxiedCopies(id) + if err != nil { + return + } + running := map[string]bool{} + for _, c := range status { + running[c.Slug] = copyRunning(c) + } + var ownIP string + for _, r := range records { + if !r.StoppedByCap || running[r.Slug] { + continue + } + if ownIP == "" { + ownIP = detectOwnIP(ctx) + } + if _, err := p.StartProxied(ctx, r.Slug, id, m.stopTimeout(r.Slug), specOf(d), ownIP); err != nil { + m.store.RecordEvent(r.Slug, "proxied_start_error", maskSecrets(err.Error(), d)) + continue + } + _ = m.store.SetProxiedCopyCapped(id, r.Slug, false) + } +} + func (m *Manager) stopCapped(ctx context.Context, p runtime.ProxyProvider, id int64, d proxies.Definition, targets map[string]bool) { slugs := make([]string, 0, len(targets)) for slug := range targets { diff --git a/internal/services/proxies_test.go b/internal/services/proxies_test.go index c17b5f3..d76acd6 100644 --- a/internal/services/proxies_test.go +++ b/internal/services/proxies_test.go @@ -28,6 +28,8 @@ type fakeProxyProvider struct { specs []runtime.ProxySpec removed []string allGone []string + // startFails makes that many StartProxied calls fail before they succeed again. + startFails int } func newFakeProxyProvider() *fakeProxyProvider { @@ -52,6 +54,10 @@ func (f *fakeProxyProvider) StopProxied(_ context.Context, slug string, id int64 } func (f *fakeProxyProvider) StartProxied(_ context.Context, slug string, id int64, _ int, spec runtime.ProxySpec, _ string) (runtime.Exit, error) { f.starts = append(f.starts, key(slug, id)) + if f.startFails > 0 { + f.startFails-- + return runtime.Exit{}, errors.New("daemon busy") + } f.specs = append(f.specs, spec) f.running[key(slug, id)] = true return runtime.Exit{IP: "203.0.113.7"}, nil @@ -414,3 +420,42 @@ func TestANewProxyWithAResetDayStartsItsPeriod(t *testing.T) { t.Fatalf("period: %+v %v", s, err) } } + +// A reset whose start failed left the copy marked as the cap's and stopped until the +// next reset, a month later. Every pass now tries again while the proxy is uncapped. +func TestACopyAResetCouldNotStartIsStartedOnTheNextPass(t *testing.T) { + m, p, st := newProxyManager(t) + ctx := context.Background() + deployExample(t, st) + view, _ := m.SaveProxy(ctx, 0, map[string]string{"name": "Gate", "kind": "socks5", "host": "203.0.113.50", "port": "7000", "cap_gb": "1"}) + if _, err := m.DeployProxied(ctx, "example", view.ID); err != nil { + t.Fatal(err) + } + k := key("example", view.ID) + p.rx[k] = 950_000_000 + if _, err := m.ProxyTick(ctx); err != nil || p.running[k] { + t.Fatalf("capped: %v", err) + } + p.startFails = 1 + if started, _ := m.ResetProxy(ctx, view.ID); len(started) != 0 || p.running[k] { + t.Fatalf("the reset's start was meant to fail: %v", started) + } + if _, err := m.ProxyTick(ctx); err != nil { + t.Fatal(err) + } + if !p.running[k] { + t.Fatal("the next pass must start the copy the reset could not") + } + records, _ := st.ListProxiedCopies(view.ID) + if len(records) != 1 || records[0].StoppedByCap { + t.Fatalf("the mark is cleared once it runs: %+v", records) + } + // A copy the owner stops afterwards stays stopped on the next pass. + if err := m.StopProxied(ctx, "example", view.ID); err != nil { + t.Fatal(err) + } + p.starts = nil + if _, err := m.ProxyTick(ctx); err != nil || len(p.starts) != 0 { + t.Fatalf("a copy the owner stopped was started: %v %v", p.starts, err) + } +} From 67d75c46e6563527046f7e60fae601307e4307a6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:14:31 +0100 Subject: [PATCH 2/6] fix(proxies): clear a stale cap mark on a running copy, and record a failed clear --- internal/services/proxies.go | 15 +++++++++++++-- internal/services/proxies_test.go | 11 +++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/internal/services/proxies.go b/internal/services/proxies.go index 69126ac..0801934 100644 --- a/internal/services/proxies.go +++ b/internal/services/proxies.go @@ -735,7 +735,15 @@ func (m *Manager) restartCapStoppedDown(ctx context.Context, p runtime.ProxyProv } var ownIP string for _, r := range records { - if !r.StoppedByCap || running[r.Slug] { + if !r.StoppedByCap { + continue + } + if running[r.Slug] { + // Started, but clearing its mark failed: clear it now, or a later crash + // of the copy would be read as the cap's stop. + if err := m.store.SetProxiedCopyCapped(id, r.Slug, false); err != nil { + m.store.RecordEvent(r.Slug, "proxied_mark_error", err.Error()) + } continue } if ownIP == "" { @@ -745,7 +753,10 @@ func (m *Manager) restartCapStoppedDown(ctx context.Context, p runtime.ProxyProv m.store.RecordEvent(r.Slug, "proxied_start_error", maskSecrets(err.Error(), d)) continue } - _ = m.store.SetProxiedCopyCapped(id, r.Slug, false) + if err := m.store.SetProxiedCopyCapped(id, r.Slug, false); err != nil { + // The next pass sees it running and marked, and clears it then. + m.store.RecordEvent(r.Slug, "proxied_mark_error", err.Error()) + } } } diff --git a/internal/services/proxies_test.go b/internal/services/proxies_test.go index d76acd6..e7c4442 100644 --- a/internal/services/proxies_test.go +++ b/internal/services/proxies_test.go @@ -450,6 +450,17 @@ func TestACopyAResetCouldNotStartIsStartedOnTheNextPass(t *testing.T) { if len(records) != 1 || records[0].StoppedByCap { t.Fatalf("the mark is cleared once it runs: %+v", records) } + // A running copy left marked (its clear failed) has the mark cleared on a pass. + if err := st.SetProxiedCopyCapped(view.ID, "example", true); err != nil { + t.Fatal(err) + } + p.starts = nil + if _, err := m.ProxyTick(ctx); err != nil || len(p.starts) != 0 { + t.Fatalf("a running marked copy must not be started: %v %v", p.starts, err) + } + if records, _ := st.ListProxiedCopies(view.ID); records[0].StoppedByCap { + t.Fatal("the stale mark on a running copy is cleared") + } // A copy the owner stops afterwards stays stopped on the next pass. if err := m.StopProxied(ctx, "example", view.ID); err != nil { t.Fatal(err) From 1352704c6668a0e06ec6e189d807a82616cd074c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:31:46 +0100 Subject: [PATCH 3/6] fix(proxies): keep the cap mark while only the tunnel of a copy runs --- internal/services/proxies.go | 4 +++- internal/services/proxies_test.go | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/services/proxies.go b/internal/services/proxies.go index 0801934..922c23e 100644 --- a/internal/services/proxies.go +++ b/internal/services/proxies.go @@ -729,9 +729,11 @@ func (m *Manager) restartCapStoppedDown(ctx context.Context, p runtime.ProxyProv if err != nil { return } + // Only a running earner counts: a tunnel left up by a start that timed out is not + // a copy that runs, so its mark stays and the next pass tries again. running := map[string]bool{} for _, c := range status { - running[c.Slug] = copyRunning(c) + running[c.Slug] = c.Status == "running" || c.Status == "restarting" } var ownIP string for _, r := range records { diff --git a/internal/services/proxies_test.go b/internal/services/proxies_test.go index e7c4442..bd0cdae 100644 --- a/internal/services/proxies_test.go +++ b/internal/services/proxies_test.go @@ -30,6 +30,8 @@ type fakeProxyProvider struct { allGone []string // startFails makes that many StartProxied calls fail before they succeed again. startFails int + // earnerDown reports that copy's earner stopped while its tunnel runs. + earnerDown map[string]bool } func newFakeProxyProvider() *fakeProxyProvider { @@ -81,7 +83,11 @@ func (f *fakeProxyProvider) ProxiedStatus(context.Context) ([]runtime.ProxiedCop status = "running" } rx, tx := f.rx[k], uint64(0) - out = append(out, runtime.ProxiedCopy{Slug: slug, ProxyID: id, Status: status, TunnelStatus: status, TunnelContainerID: "t-" + k, TunnelStartedAt: "s", NetRxBytes: &rx, NetTxBytes: &tx, ExitIP: "203.0.113.7"}) + earner := status + if f.earnerDown[k] { + earner = "exited" + } + out = append(out, runtime.ProxiedCopy{Slug: slug, ProxyID: id, Status: earner, TunnelStatus: status, TunnelContainerID: "t-" + k, TunnelStartedAt: "s", NetRxBytes: &rx, NetTxBytes: &tx, ExitIP: "203.0.113.7"}) } return out, nil } @@ -461,6 +467,17 @@ func TestACopyAResetCouldNotStartIsStartedOnTheNextPass(t *testing.T) { if records, _ := st.ListProxiedCopies(view.ID); records[0].StoppedByCap { t.Fatal("the stale mark on a running copy is cleared") } + // A tunnel left up by a start that timed out, with the earner down, keeps the mark + // and is started again. + if err := st.SetProxiedCopyCapped(view.ID, "example", true); err != nil { + t.Fatal(err) + } + p.earnerDown = map[string]bool{k: true} + p.starts = nil + if _, err := m.ProxyTick(ctx); err != nil || len(p.starts) != 1 { + t.Fatalf("a copy with only its tunnel up must be started again: %v %v", p.starts, err) + } + p.earnerDown = nil // A copy the owner stops afterwards stays stopped on the next pass. if err := m.StopProxied(ctx, "example", view.ID); err != nil { t.Fatal(err) From ebeb8bf0a6eb9e1348429c8cbec9a554ab98b061 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:43:09 +0100 Subject: [PATCH 4/6] fix(proxies): order the cap retry with an owner's start or stop, so a returned stop stands --- internal/services/proxies.go | 35 +++++++++++++++++++++++++++++-- internal/services/proxies_test.go | 23 ++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/internal/services/proxies.go b/internal/services/proxies.go index 922c23e..15fb731 100644 --- a/internal/services/proxies.go +++ b/internal/services/proxies.go @@ -48,6 +48,10 @@ var detectOwnIP = func(ctx context.Context) string { // write that one column, and two that overlap would lose one write. type proxyState struct { proxyMu sync.Mutex + // ownerMu orders an owner's Start or Stop of a copy with the tick's retry of a + // copy the cap stopped, so a stop that returned is never undone by a retry that + // read the mark before it. + ownerMu sync.Mutex // last is the latest status read, for the UI. lastMu sync.Mutex last []runtime.ProxiedCopy @@ -470,6 +474,8 @@ func (m *Manager) otherExits(self int64) []proxies.Exit { // StopProxied stops a proxied copy. A stop by hand is the owner's decision, so a // later cap reset does not start it again. func (m *Manager) StopProxied(ctx context.Context, slug string, id int64) error { + m.proxy.ownerMu.Lock() + defer m.proxy.ownerMu.Unlock() p, err := m.proxier() if err != nil { return err @@ -487,6 +493,8 @@ func (m *Manager) StopProxied(ctx context.Context, slug string, id int64) error // StartProxied starts a proxied copy, refused while its proxy is capped. func (m *Manager) StartProxied(ctx context.Context, slug string, id int64) error { + m.proxy.ownerMu.Lock() + defer m.proxy.ownerMu.Unlock() d, s, err := m.loadProxy(id) if err != nil { return err @@ -751,8 +759,7 @@ func (m *Manager) restartCapStoppedDown(ctx context.Context, p runtime.ProxyProv if ownIP == "" { ownIP = detectOwnIP(ctx) } - if _, err := p.StartProxied(ctx, r.Slug, id, m.stopTimeout(r.Slug), specOf(d), ownIP); err != nil { - m.store.RecordEvent(r.Slug, "proxied_start_error", maskSecrets(err.Error(), d)) + if !m.retryCapStart(ctx, p, id, d, r.Slug, ownIP) { continue } if err := m.store.SetProxiedCopyCapped(id, r.Slug, false); err != nil { @@ -762,6 +769,30 @@ func (m *Manager) restartCapStoppedDown(ctx context.Context, p runtime.ProxyProv } } +// retryCapStart starts one cap-stopped copy, unless an owner's Start or Stop got to +// it first: the mark is read again under ownerMu, which those hold for their whole +// run. It reports whether the copy was started. +func (m *Manager) retryCapStart(ctx context.Context, p runtime.ProxyProvider, id int64, d proxies.Definition, slug, ownIP string) bool { + m.proxy.ownerMu.Lock() + defer m.proxy.ownerMu.Unlock() + records, err := m.store.ListProxiedCopies(id) + if err != nil { + return false + } + marked := false + for _, r := range records { + marked = marked || (r.Slug == slug && r.StoppedByCap) + } + if !marked { + return false + } + if _, err := p.StartProxied(ctx, slug, id, m.stopTimeout(slug), specOf(d), ownIP); err != nil { + m.store.RecordEvent(slug, "proxied_start_error", maskSecrets(err.Error(), d)) + return false + } + return true +} + func (m *Manager) stopCapped(ctx context.Context, p runtime.ProxyProvider, id int64, d proxies.Definition, targets map[string]bool) { slugs := make([]string, 0, len(targets)) for slug := range targets { diff --git a/internal/services/proxies_test.go b/internal/services/proxies_test.go index bd0cdae..03c84ae 100644 --- a/internal/services/proxies_test.go +++ b/internal/services/proxies_test.go @@ -487,3 +487,26 @@ func TestACopyAResetCouldNotStartIsStartedOnTheNextPass(t *testing.T) { t.Fatalf("a copy the owner stopped was started: %v %v", p.starts, err) } } + +// A retry that picked a copy before the owner stopped it reads the mark again under +// the lock the owner's Stop holds, so the stop that returned stands. +func TestARetryNeverUndoesAnOwnerStop(t *testing.T) { + m, p, st := newProxyManager(t) + ctx := context.Background() + deployExample(t, st) + view, _ := m.SaveProxy(ctx, 0, map[string]string{"name": "Gate", "kind": "socks5", "host": "203.0.113.50", "port": "7000"}) + if _, err := m.DeployProxied(ctx, "example", view.ID); err != nil { + t.Fatal(err) + } + if err := st.SetProxiedCopyCapped(view.ID, "example", true); err != nil { + t.Fatal(err) + } + if err := m.StopProxied(ctx, "example", view.ID); err != nil { + t.Fatal(err) + } + d, _, _ := m.loadProxy(view.ID) + p.starts = nil + if m.retryCapStart(ctx, p, view.ID, d, "example", "192.0.2.1") || len(p.starts) != 0 { + t.Fatalf("the retry started a copy the owner stopped: %v", p.starts) + } +} From 61b221ff4cb5f9acfa66eb26f45ea254b6810d70 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:57:35 +0100 Subject: [PATCH 5/6] fix(proxies): lock per copy instead of globally, reload the proxy before a cap retry An owner Start or Stop, a reset's start and the tick's retry of one copy now take that copy's own lock, so unrelated copies never wait on a slow retry. The retry reloads the proxy under the lock (new settings reach the copy, a returned cap stops it), and the own-IP lookup runs once per pass. --- internal/services/proxies.go | 61 +++++++++++++++++++++---------- internal/services/proxies_test.go | 15 +++++++- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/internal/services/proxies.go b/internal/services/proxies.go index 15fb731..03c2f15 100644 --- a/internal/services/proxies.go +++ b/internal/services/proxies.go @@ -48,10 +48,11 @@ var detectOwnIP = func(ctx context.Context) string { // write that one column, and two that overlap would lose one write. type proxyState struct { proxyMu sync.Mutex - // ownerMu orders an owner's Start or Stop of a copy with the tick's retry of a - // copy the cap stopped, so a stop that returned is never undone by a retry that - // read the mark before it. - ownerMu sync.Mutex + // copyMu holds one mutex per copy ("id/slug"). It orders an owner's Start or Stop, + // a reset's start and the tick's retry of the same copy, so a stop that returned + // is never undone by a retry that read the mark before it, and two starts never + // overlap. Other copies never wait on it. + copyMu sync.Map // last is the latest status read, for the UI. lastMu sync.Mutex last []runtime.ProxiedCopy @@ -474,8 +475,9 @@ func (m *Manager) otherExits(self int64) []proxies.Exit { // StopProxied stops a proxied copy. A stop by hand is the owner's decision, so a // later cap reset does not start it again. func (m *Manager) StopProxied(ctx context.Context, slug string, id int64) error { - m.proxy.ownerMu.Lock() - defer m.proxy.ownerMu.Unlock() + mu := m.copyLock(id, slug) + mu.Lock() + defer mu.Unlock() p, err := m.proxier() if err != nil { return err @@ -493,8 +495,9 @@ func (m *Manager) StopProxied(ctx context.Context, slug string, id int64) error // StartProxied starts a proxied copy, refused while its proxy is capped. func (m *Manager) StartProxied(ctx context.Context, slug string, id int64) error { - m.proxy.ownerMu.Lock() - defer m.proxy.ownerMu.Unlock() + mu := m.copyLock(id, slug) + mu.Lock() + defer mu.Unlock() d, s, err := m.loadProxy(id) if err != nil { return err @@ -582,11 +585,17 @@ func (m *Manager) releaseCapped(ctx context.Context, id int64, d proxies.Definit if !c.StoppedByCap { continue } - if _, err := p.StartProxied(ctx, c.Slug, id, m.stopTimeout(c.Slug), specOf(d), ownIP); err != nil { + mu := m.copyLock(id, c.Slug) + mu.Lock() + _, err := p.StartProxied(ctx, c.Slug, id, m.stopTimeout(c.Slug), specOf(d), ownIP) + if err == nil { + _ = m.store.SetProxiedCopyCapped(id, c.Slug, false) + } + mu.Unlock() + if err != nil { m.store.RecordEvent(c.Slug, "proxied_start_error", maskSecrets(err.Error(), d)) continue } - _ = m.store.SetProxiedCopyCapped(id, c.Slug, false) started = append(started, c.Slug) } return started, nil @@ -744,6 +753,7 @@ func (m *Manager) restartCapStoppedDown(ctx context.Context, p runtime.ProxyProv running[c.Slug] = c.Status == "running" || c.Status == "restarting" } var ownIP string + detected := false // one lookup per pass, even when it fails for _, r := range records { if !r.StoppedByCap { continue @@ -756,10 +766,10 @@ func (m *Manager) restartCapStoppedDown(ctx context.Context, p runtime.ProxyProv } continue } - if ownIP == "" { - ownIP = detectOwnIP(ctx) + if !detected { + ownIP, detected = detectOwnIP(ctx), true } - if !m.retryCapStart(ctx, p, id, d, r.Slug, ownIP) { + if !m.retryCapStart(ctx, p, id, r.Slug, ownIP) { continue } if err := m.store.SetProxiedCopyCapped(id, r.Slug, false); err != nil { @@ -769,12 +779,25 @@ func (m *Manager) restartCapStoppedDown(ctx context.Context, p runtime.ProxyProv } } -// retryCapStart starts one cap-stopped copy, unless an owner's Start or Stop got to -// it first: the mark is read again under ownerMu, which those hold for their whole -// run. It reports whether the copy was started. -func (m *Manager) retryCapStart(ctx context.Context, p runtime.ProxyProvider, id int64, d proxies.Definition, slug, ownIP string) bool { - m.proxy.ownerMu.Lock() - defer m.proxy.ownerMu.Unlock() +// copyLock is the mutex of one copy; see proxyState.copyMu. +func (m *Manager) copyLock(id int64, slug string) *sync.Mutex { + mu, _ := m.proxy.copyMu.LoadOrStore(fmt.Sprintf("%d/%s", id, slug), &sync.Mutex{}) + return mu.(*sync.Mutex) +} + +// retryCapStart starts one cap-stopped copy, unless an owner's Start or Stop, or a +// reset, got to it first: the mark is read again under the copy's lock, which those +// hold for their whole run. It reports whether the copy was started. +func (m *Manager) retryCapStart(ctx context.Context, p runtime.ProxyProvider, id int64, slug, ownIP string) bool { + mu := m.copyLock(id, slug) + mu.Lock() + defer mu.Unlock() + // The proxy as stored now: an edit while the retry waited reaches the copy, and a + // cap that came back stops the retry. + d, s, err := m.loadProxy(id) + if err != nil || s.CappedAt != "" { + return false + } records, err := m.store.ListProxiedCopies(id) if err != nil { return false diff --git a/internal/services/proxies_test.go b/internal/services/proxies_test.go index 03c84ae..4a091d9 100644 --- a/internal/services/proxies_test.go +++ b/internal/services/proxies_test.go @@ -504,9 +504,20 @@ func TestARetryNeverUndoesAnOwnerStop(t *testing.T) { if err := m.StopProxied(ctx, "example", view.ID); err != nil { t.Fatal(err) } - d, _, _ := m.loadProxy(view.ID) p.starts = nil - if m.retryCapStart(ctx, p, view.ID, d, "example", "192.0.2.1") || len(p.starts) != 0 { + if m.retryCapStart(ctx, p, view.ID, "example", "192.0.2.1") || len(p.starts) != 0 { t.Fatalf("the retry started a copy the owner stopped: %v", p.starts) } + // A cap that came back while the retry waited stops it too. + if err := st.SetProxiedCopyCapped(view.ID, "example", true); err != nil { + t.Fatal(err) + } + m.proxy.proxyMu.Lock() + _, s, _ := m.loadProxy(view.ID) + s.CappedAt, s.CapReason = "2026-11-07T00:00:00Z", "Gate reached its cap" + _ = m.saveState(view.ID, s) + m.proxy.proxyMu.Unlock() + if m.retryCapStart(ctx, p, view.ID, "example", "192.0.2.1") || len(p.starts) != 0 { + t.Fatalf("the retry started a copy while the cap holds: %v", p.starts) + } } From 4476375da8e2e08777115f0b0d3de595abfa506f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:11:37 +0100 Subject: [PATCH 6/6] fix(proxies): clear the retry's cap mark inside the copy lock that cap stops also take --- internal/services/proxies.go | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/internal/services/proxies.go b/internal/services/proxies.go index 03c2f15..cb3cb21 100644 --- a/internal/services/proxies.go +++ b/internal/services/proxies.go @@ -769,13 +769,7 @@ func (m *Manager) restartCapStoppedDown(ctx context.Context, p runtime.ProxyProv if !detected { ownIP, detected = detectOwnIP(ctx), true } - if !m.retryCapStart(ctx, p, id, r.Slug, ownIP) { - continue - } - if err := m.store.SetProxiedCopyCapped(id, r.Slug, false); err != nil { - // The next pass sees it running and marked, and clears it then. - m.store.RecordEvent(r.Slug, "proxied_mark_error", err.Error()) - } + m.retryCapStart(ctx, p, id, r.Slug, ownIP) } } @@ -813,6 +807,18 @@ func (m *Manager) retryCapStart(ctx context.Context, p runtime.ProxyProvider, id m.store.RecordEvent(slug, "proxied_start_error", maskSecrets(err.Error(), d)) return false } + // Cleared inside the copy's lock, which stopCapped also takes, and only while the + // cap is still lifted: a cap that came back during the start keeps the mark, so + // the next stop and reset still treat the copy as the cap's. + m.proxy.proxyMu.Lock() + _, now, err := m.loadProxy(id) + m.proxy.proxyMu.Unlock() + if err == nil && now.CappedAt == "" { + if err := m.store.SetProxiedCopyCapped(id, slug, false); err != nil { + // The next pass sees it running and marked, and clears it then. + m.store.RecordEvent(slug, "proxied_mark_error", err.Error()) + } + } return true } @@ -823,6 +829,17 @@ func (m *Manager) stopCapped(ctx context.Context, p runtime.ProxyProvider, id in } sort.Strings(slugs) for _, slug := range slugs { + m.stopCappedCopy(ctx, p, id, d, slug) + } +} + +// stopCappedCopy stops one copy at the cap under that copy's lock, so a retry's +// start and mark clear never interleave with the stop and its mark. +func (m *Manager) stopCappedCopy(ctx context.Context, p runtime.ProxyProvider, id int64, d proxies.Definition, slug string) { + mu := m.copyLock(id, slug) + mu.Lock() + defer mu.Unlock() + { wasRunning, err := p.StopProxied(ctx, slug, id, m.stopTimeout(slug)) m.proxy.lastMu.Lock() if m.proxy.unstopped == nil { @@ -840,7 +857,7 @@ func (m *Manager) stopCapped(ctx context.Context, p runtime.ProxyProvider, id in } m.proxy.lastMu.Unlock() if err != nil || !wasRunning { - continue + return } // A reset or a cap change may have lifted the cap while the stop ran; then // the copy goes back up instead of being marked.