From bdd7f7b0f3631a4a6e90dd54a98ec5d99b7c6287 Mon Sep 17 00:00:00 2001 From: squarepots <46488165+squarepots@users.noreply.github.com> Date: Fri, 18 Sep 2026 01:09:38 +0800 Subject: [PATCH 1/6] Recover verified deployments without redeploying Signed-off-by: squarepots <46488165+squarepots@users.noreply.github.com> --- internal/steward/execution.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/steward/execution.go b/internal/steward/execution.go index 91a7460..d7857dc 100644 --- a/internal/steward/execution.go +++ b/internal/steward/execution.go @@ -59,9 +59,6 @@ func deployRouteWithoutRender(ctx context.Context, state *State, routeID string) } func deploymentAuditAllowsMutation(route *Route, current AuditEvidence) error { - if current.Category == "in-sync" { - return nil - } if current.Category == "service-missing" && route.State != "deployed" { return nil } @@ -85,6 +82,9 @@ func deployRoute(ctx context.Context, state *State, routeID string, skipClientVa } current := AuditRoute(ctx, state, routeID) + if current.Category == "in-sync" { + return adoptVerifiedRoute(state, route, current, skipClientValidation, renderClients) + } if err := deploymentAuditAllowsMutation(route, current); err != nil { return nil, &operationStageError{Stage: "remote-preflight-audit", StateChanged: "remote-state-unchanged", Retry: "audit", Err: err} } From 9ae0be31e72d2de069fb3fa2a5c8fdf451c6d88c Mon Sep 17 00:00:00 2001 From: squarepots <46488165+squarepots@users.noreply.github.com> Date: Fri, 18 Sep 2026 01:10:02 +0800 Subject: [PATCH 2/6] Cover verified deployment adoption Signed-off-by: squarepots <46488165+squarepots@users.noreply.github.com> --- internal/steward/deployment_recovery_test.go | 45 ++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/internal/steward/deployment_recovery_test.go b/internal/steward/deployment_recovery_test.go index 261bfaa..84bbc39 100644 --- a/internal/steward/deployment_recovery_test.go +++ b/internal/steward/deployment_recovery_test.go @@ -6,13 +6,13 @@ import ( "testing" ) -func TestDeploymentAuditAllowsOnlyKnownRemoteStates(t *testing.T) { +func TestDeploymentAuditAllowsOnlyKnownMissingRemoteStateForMutation(t *testing.T) { route := &Route{State: "pending"} if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "service-missing"}); err != nil { t.Fatalf("fresh pending Route rejected service-missing state: %v", err) } - if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "in-sync"}); err != nil { - t.Fatalf("pending Route rejected verified in-sync state: %v", err) + if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "in-sync"}); err == nil { + t.Fatal("pending Route accepted remote mutation even though the verified deployment is already in sync") } if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "undetermined"}); err == nil { t.Fatal("pending Route accepted undetermined remote state") @@ -29,6 +29,45 @@ func TestDeploymentAuditAllowsOnlyKnownRemoteStates(t *testing.T) { } } +func TestAdoptVerifiedRouteRecoversLocalStateWithoutDeployment(t *testing.T) { + state, route := healthFixture(t, "direct", false) + current := findRoute(state.Inventory, route.ID) + current.Enabled = false + current.State = "deploying" + if err := state.Save(false); err != nil { + t.Fatal(err) + } + + exit := findServer(state.Inventory, route.ExitServer) + if exit == nil { + t.Fatal("fixture is missing Route exit Server") + } + actual := exit.Network.ExpectedEgressIPv4 + evidence := AuditEvidence{ + Route: route.ID, + Status: "healthy", + Category: "in-sync", + ActualEgressIPv4: &actual, + EgressMatchesDeclaredExit: true, + } + + result, err := adoptVerifiedRoute(state, current, evidence, true, false) + if err != nil { + t.Fatal(err) + } + if result["state"] != "deployed" || result["enabled"] != true { + t.Fatalf("verified deployment was not adopted: %#v", result) + } + reloaded, err := LoadState(state.PrivateDir) + if err != nil { + t.Fatal(err) + } + recovered := findRoute(reloaded.Inventory, route.ID) + if recovered == nil || recovered.State != "deployed" || !recovered.Enabled { + t.Fatalf("verified remote state was not committed locally: %#v", recovered) + } +} + func TestMarkRouteDeployingPersistsRetryCheckpoint(t *testing.T) { state, route := healthFixture(t, "direct", false) current := findRoute(state.Inventory, route.ID) From 5b2a09faf1afe23f09d53270fffa88cdb0bbe5cb Mon Sep 17 00:00:00 2001 From: squarepots <46488165+squarepots@users.noreply.github.com> Date: Fri, 18 Sep 2026 01:10:10 +0800 Subject: [PATCH 3/6] Bump patch version for deployment recovery Signed-off-by: squarepots <46488165+squarepots@users.noreply.github.com> --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index b1b25a5..5859406 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -2.2.2 +2.2.3 From 7d37a21838ab4549b04e4b721727728e739af37e Mon Sep 17 00:00:00 2001 From: squarepots <46488165+squarepots@users.noreply.github.com> Date: Fri, 18 Sep 2026 01:12:58 +0800 Subject: [PATCH 4/6] Scope in-sync recovery to deployment retries Signed-off-by: squarepots <46488165+squarepots@users.noreply.github.com> --- internal/steward/execution.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/steward/execution.go b/internal/steward/execution.go index d7857dc..438447b 100644 --- a/internal/steward/execution.go +++ b/internal/steward/execution.go @@ -59,6 +59,9 @@ func deployRouteWithoutRender(ctx context.Context, state *State, routeID string) } func deploymentAuditAllowsMutation(route *Route, current AuditEvidence) error { + if current.Category == "in-sync" && route.State == "pending" { + return nil + } if current.Category == "service-missing" && route.State != "deployed" { return nil } @@ -82,7 +85,7 @@ func deployRoute(ctx context.Context, state *State, routeID string, skipClientVa } current := AuditRoute(ctx, state, routeID) - if current.Category == "in-sync" { + if current.Category == "in-sync" && (route.State == "deploying" || route.State == "deployed") { return adoptVerifiedRoute(state, route, current, skipClientValidation, renderClients) } if err := deploymentAuditAllowsMutation(route, current); err != nil { @@ -178,7 +181,7 @@ func operateDirect(ctx context.Context, state *State, route Route, server Server if _, err = runSCP(ctx, host, []string{"-r", filepath.Join(localRoot, "server"), host.Address + ":" + remoteRoot + "/"}); err != nil { return nil, err } - remoteServer := remoteRoot + "/server" + remoteServer := "/tmp/route-steward-" + strings.TrimPrefix(remoteRoot, "/tmp/route-steward-") + "/server" if _, err = runSSH(ctx, host, bashCommand("sudo", "bash", remoteServer+"/preflight.sh")); err != nil { return nil, err } From 13a154105f5141f022dc92c713e027ed76bef1ea Mon Sep 17 00:00:00 2001 From: squarepots <46488165+squarepots@users.noreply.github.com> Date: Fri, 18 Sep 2026 01:13:14 +0800 Subject: [PATCH 5/6] Test state-sensitive deployment recovery Signed-off-by: squarepots <46488165+squarepots@users.noreply.github.com> --- internal/steward/deployment_recovery_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/steward/deployment_recovery_test.go b/internal/steward/deployment_recovery_test.go index 84bbc39..fe0a16f 100644 --- a/internal/steward/deployment_recovery_test.go +++ b/internal/steward/deployment_recovery_test.go @@ -6,13 +6,13 @@ import ( "testing" ) -func TestDeploymentAuditAllowsOnlyKnownMissingRemoteStateForMutation(t *testing.T) { +func TestDeploymentAuditAllowsMutationOnlyFromKnownSafeStates(t *testing.T) { route := &Route{State: "pending"} if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "service-missing"}); err != nil { t.Fatalf("fresh pending Route rejected service-missing state: %v", err) } - if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "in-sync"}); err == nil { - t.Fatal("pending Route accepted remote mutation even though the verified deployment is already in sync") + if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "in-sync"}); err != nil { + t.Fatalf("fresh pending Route rejected verified in-sync state: %v", err) } if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "undetermined"}); err == nil { t.Fatal("pending Route accepted undetermined remote state") @@ -22,11 +22,17 @@ func TestDeploymentAuditAllowsOnlyKnownMissingRemoteStateForMutation(t *testing. if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "service-missing"}); err != nil { t.Fatalf("retry checkpoint rejected known missing service: %v", err) } + if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "in-sync"}); err == nil { + t.Fatal("deploying Route accepted another remote mutation after audit proved the existing deployment is in sync") + } route.State = "deployed" if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "service-missing"}); err == nil { t.Fatal("deployed Route accepted a missing remote service") } + if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "in-sync"}); err == nil { + t.Fatal("deployed Route accepted another remote mutation after audit proved the existing deployment is in sync") + } } func TestAdoptVerifiedRouteRecoversLocalStateWithoutDeployment(t *testing.T) { From 18718c67561c6ae377327ef6d4c18014133df789 Mon Sep 17 00:00:00 2001 From: squarepots <46488165+squarepots@users.noreply.github.com> Date: Fri, 18 Sep 2026 01:14:25 +0800 Subject: [PATCH 6/6] Keep deployment recovery diff focused Signed-off-by: squarepots <46488165+squarepots@users.noreply.github.com> --- internal/steward/execution.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/steward/execution.go b/internal/steward/execution.go index 438447b..d17eda6 100644 --- a/internal/steward/execution.go +++ b/internal/steward/execution.go @@ -181,7 +181,7 @@ func operateDirect(ctx context.Context, state *State, route Route, server Server if _, err = runSCP(ctx, host, []string{"-r", filepath.Join(localRoot, "server"), host.Address + ":" + remoteRoot + "/"}); err != nil { return nil, err } - remoteServer := "/tmp/route-steward-" + strings.TrimPrefix(remoteRoot, "/tmp/route-steward-") + "/server" + remoteServer := remoteRoot + "/server" if _, err = runSSH(ctx, host, bashCommand("sudo", "bash", remoteServer+"/preflight.sh")); err != nil { return nil, err }