fix(proxies): keep the running copy when a redeploy's tunnel cannot start

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- **Redeploying a proxied copy could leave the proxy with no copy at all.** The old earner and tunnel were removed before the new tunnel started, so a tunnel that could not start (an image missing locally, a proxy that does not answer) cost the running copy. The new tunnel now starts under a staged name next to the running copy, and the old pair goes only after the new tunnel has proved an exit.
 - **The Fleet page could draw over another view, and lose a half-typed proxy.** Fleet reads the backend twice before it draws, so a user who opened another view meanwhile got the Fleet page back. Any proxy action redraws the page, which also reset the open proxy editor and dropped unsaved values, a typed password included. A redraw now gives up when Fleet is no longer shown or a newer redraw started, and keeps the open editor when the same proxy's editor is drawn again.
 - **The Proxies section had no help link.** It now links to the published page on proxied copies, https://geiserx.github.io/CashPilot/proxies/.
 - **A proxied copy's stop that failed halfway could be undone.** The earner stopped first. If the tunnel then failed to stop, the next status read saw a ready tunnel beside a stopped earner and started the earner again. The tunnel now stops first, so a half-finished stop never leaves that state.

internal/runtime/proxied.go

@@ -36,6 +36,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/GeiserX/CashPilot-Desktop/internal/catalog"
@@ -212,6 +213,25 @@ func tunnelName(slug string, proxyID int64) string {
 	return fmt.Sprintf("%stunnel-%s-p%d", containerName(""), slug, proxyID)
 }
 
+// stagedSuffix: a redeploy starts the new tunnel under its name plus this suffix,
+// next to the copy that runs now, and gives it the final name only once the old pair
+// is gone. The status read never reports a staged tunnel.
+const stagedSuffix = "-next"
+
+// proxiedSwap makes deployProxied's swap of the old pair for the staged tunnel, and
+// the status read, take turns: a status read must not start an earner that is being
+// removed, or read a half-swapped pair.
+var proxiedSwap sync.Mutex
+
+// proxiedDeploys runs one deploy per copy at a time: a second one would take the
+// first one's staged tunnel for a leftover and remove it.
+var proxiedDeploys sync.Map // "slug/id" -> *sync.Mutex
+
+func deployLock(slug string, proxyID int64) *sync.Mutex {
+	mu, _ := proxiedDeploys.LoadOrStore(fmt.Sprintf("%s/%d", slug, proxyID), &sync.Mutex{})
+	return mu.(*sync.Mutex)
+}
+
 func proxiedName(slug string, proxyID int64) string {
 	return fmt.Sprintf("%s-p%d", containerName(slug), proxyID)
 }
@@ -395,7 +415,8 @@ func sidecarCreate(name, slug string, proxyID *int64, image, network string) cli
 	if proxyID == nil {
 		restart = container.RestartPolicy{Name: "no"}
 	}
-	hostname := strings.TrimPrefix(name, containerName(""))
+	// A staged tunnel already gets the hostname of the name it is promoted to.
+	hostname := strings.TrimSuffix(strings.TrimPrefix(name, containerName("")), stagedSuffix)
 	if len(hostname) > 63 {
 		hostname = hostname[:63]
 	}
@@ -500,6 +521,7 @@ type tunnelDocker interface {
 	stop(ctx context.Context, id string, timeout int) error
 	restart(ctx context.Context, id string, timeout int) error
 	remove(ctx context.Context, id string) error
+	rename(ctx context.Context, id, name string) error
 	inspect(ctx context.Context, nameOrID string) (container.InspectResponse, bool, error)
 	list(ctx context.Context, labels ...string) ([]container.Summary, error)
 	readFile(ctx context.Context, id, path string) (string, error)
@@ -560,6 +582,10 @@ func (d liveTunnelDocker) remove(ctx context.Context, id string) error {
 	_, err := d.cli.ContainerRemove(ctx, id, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
 	return err
 }
+func (d liveTunnelDocker) rename(ctx context.Context, id, name string) error {
+	_, err := d.cli.ContainerRename(ctx, id, client.ContainerRenameOptions{NewName: name})
+	return err
+}
 func (d liveTunnelDocker) inspect(ctx context.Context, nameOrID string) (container.InspectResponse, bool, error) {
 	res, err := d.cli.ContainerInspect(ctx, nameOrID, client.ContainerInspectOptions{})
 	if err != nil {
@@ -748,6 +774,9 @@ func pairOf(ctx context.Context, d tunnelDocker, slug string, proxyID int64) (pr
 }
 
 func deployProxied(ctx context.Context, d tunnelDocker, req ProxiedDeploy) (Exit, error) {
+	mu := deployLock(req.Slug, req.ProxyID)
+	mu.Lock()
+	defer mu.Unlock()
 	facts := d.facts(ctx)
 	if facts.OSType != "" && facts.OSType != "linux" {
 		return Exit{}, proxyErr("Proxied copies need a Linux container runtime; this one runs %s containers.", facts.OSType)
@@ -784,14 +813,40 @@ func deployProxied(ctx context.Context, d tunnelDocker, req ProxiedDeploy) (Exit
 	if err != nil {
 		return Exit{}, err
 	}
-	if _, err := removeProxied(ctx, d, req.Slug, req.ProxyID, StopTimeoutSeconds(req.Service)); err != nil {
+	pid := req.ProxyID
+	final := tunnelName(req.Slug, pid)
+	staged := final + stagedSuffix
+	// Left by a deploy that died midway; replaced only if it is ours.
+	if ins, found, err := d.inspect(ctx, staged); err != nil {
 		return Exit{}, err
+	} else if found {
+		if ins.Config == nil || !isOurs(ins.Config.Labels, req.Slug, pid) {
+			return Exit{}, proxyErr("A container named %s that CashPilot did not create is in the way", staged)
+		}
+		// One younger than the sweep age belongs to a deploy still running, in this
+		// app or another one on the same Docker host; removing it would cut that
+		// deploy's tunnel after it may already have removed the old pair.
+		if created, perr := time.Parse(time.RFC3339Nano, ins.Created); perr == nil && time.Since(created) < staleTestTunnel {
+			return Exit{}, proxyErr("Another deploy of %s through proxy %d is under way; try again in a few minutes", req.Slug, pid)
+		}
+		removeQuietly(d, ins.ID, 2)
 	}
-	pid := req.ProxyID
-	sidecarID, exit, err := startSidecar(ctx, d, tunnelName(req.Slug, pid), req.Slug, network, &pid, req.Proxy, req.OwnIP)
+	// The new tunnel proves an exit while the old pair keeps running, so a tunnel that
+	// cannot start costs nothing. Only then does the old pair go.
+	sidecarID, exit, err := startSidecar(ctx, d, staged, req.Slug, network, &pid, req.Proxy, req.OwnIP)
 	if err != nil {
 		return Exit{}, err
 	}
+	proxiedSwap.Lock()
+	_, err = removeProxied(ctx, d, req.Slug, req.ProxyID, StopTimeoutSeconds(req.Service))
+	if err == nil {
+		err = d.rename(ctx, sidecarID, final)
+	}
+	proxiedSwap.Unlock()
+	if err != nil {
+		removeQuietly(d, sidecarID, 2)
+		return Exit{}, err
+	}
 	opts, err := earnerCreate(req, sidecarID, image, facts.OSType)
 	if err != nil {
 		removeQuietly(d, sidecarID, 2)
@@ -1015,8 +1070,27 @@ const testTunnelSlug = "proxy-test"
 // app that quit during the test.
 const staleTestTunnel = 5 * time.Minute
 
-// sweepTestTunnels removes Test tunnels older than staleTestTunnel.
+// sweepTestTunnels removes Test tunnels older than staleTestTunnel, and staged
+// tunnels as old: a deploy is past its ready timeout by then, so the deploy that
+// staged it died, and it holds the proxy login while no status read reports it.
 func sweepTestTunnels(ctx context.Context, d tunnelDocker, now time.Time) {
+	if staged, err := d.list(ctx, LabelManaged+"=true", LabelRole+"="+RoleTunnel, LabelProxy); err == nil {
+		for _, c := range staged {
+			if c.Labels[LabelProxy] != "" && strings.HasSuffix(summaryName(c), stagedSuffix) && now.Sub(time.Unix(c.Created, 0)) >= staleTestTunnel {
+				id, perr := strconv.ParseInt(c.Labels[LabelProxy], 10, 64)
+				if perr != nil {
+					continue
+				}
+				// A deploy of this copy in this app may still be waiting to swap it in.
+				mu := deployLock(c.Labels[LabelService], id)
+				if !mu.TryLock() {
+					continue
+				}
+				removeQuietly(d, c.ID, 2)
+				mu.Unlock()
+			}
+		}
+	}
 	found, err := d.list(ctx, LabelManaged+"=true", LabelRole+"="+RoleTunnel, LabelService+"="+testTunnelSlug)
 	if err != nil {
 		return
@@ -1034,6 +1108,8 @@ func sweepTestTunnels(ctx context.Context, d tunnelDocker, now time.Time) {
 
 func proxiedStatus(ctx context.Context, d tunnelDocker) ([]ProxiedCopy, error) {
 	sweepTestTunnels(ctx, d, time.Now())
+	proxiedSwap.Lock()
+	defer proxiedSwap.Unlock()
 	found, err := d.list(ctx, LabelManaged+"=true", LabelProxy)
 	if err != nil {
 		return nil, err
@@ -1044,8 +1120,8 @@ func proxiedStatus(ctx context.Context, d tunnelDocker) ([]ProxiedCopy, error) {
 	}
 	pairs := map[key]map[string]container.Summary{}
 	for _, c := range found {
-		if c.Labels[LabelManaged] != "true" {
-			continue
+		if c.Labels[LabelManaged] != "true" || strings.HasSuffix(summaryName(c), stagedSuffix) {
+			continue // a staged tunnel is not part of a copy until it is promoted
 		}
 		id, err := strconv.ParseInt(c.Labels[LabelProxy], 10, 64)
 		if err != nil {