Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 58 additions & 12 deletions internal/steward/deployment_recovery_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,26 +6,72 @@ import (
"testing"
)

func TestDeploymentAuditAllowsOnlyKnownRemoteStates(t *testing.T) {
func TestDeploymentAuditDecisionUsesVerifiedRemoteState(t *testing.T) {
route := &Route{State: "pending"}
if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "service-missing"}); err != nil {
t.Fatalf("fresh pending Route rejected service-missing state: %v", err)

decision, err := deploymentAuditDecision(route, AuditEvidence{Category: "in-sync"})
if err != nil || decision != "deploy" {
t.Fatalf("fresh pending Route should still perform its initial deployment: decision=%q err=%v", decision, err)
}
if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "in-sync"}); err != nil {
t.Fatalf("pending Route rejected verified in-sync state: %v", err)

route.State = "deploying"
decision, err = deploymentAuditDecision(route, AuditEvidence{Category: "in-sync"})
if err != nil || decision != "adopt" {
t.Fatalf("verified in-sync retry should be adopted without remote mutation: decision=%q err=%v", decision, err)
}
if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "undetermined"}); err == nil {
t.Fatal("pending Route accepted undetermined remote state")

route.State = "deployed"
decision, err = deploymentAuditDecision(route, AuditEvidence{Category: "in-sync"})
if err != nil || decision != "adopt" {
t.Fatalf("already deployed in-sync Route should be adopted without remote mutation: decision=%q err=%v", decision, err)
}

route.State = "deploying"
if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "service-missing"}); err != nil {
t.Fatalf("retry checkpoint rejected known missing service: %v", err)
route.State = "pending"
decision, err = deploymentAuditDecision(route, AuditEvidence{Category: "service-missing"})
if err != nil || decision != "deploy" {
t.Fatalf("known missing service should allow deployment: decision=%q err=%v", decision, err)
}

if decision, err = deploymentAuditDecision(route, AuditEvidence{Category: "undetermined"}); err == nil || decision != "" {
t.Fatalf("undetermined remote state should block mutation: decision=%q err=%v", decision, err)
}

route.State = "deployed"
if err := deploymentAuditAllowsMutation(route, AuditEvidence{Category: "service-missing"}); err == nil {
t.Fatal("deployed Route accepted a missing remote service")
if decision, err = deploymentAuditDecision(route, AuditEvidence{Category: "service-missing"}); err == nil || decision != "" {
t.Fatalf("deployed Route with missing service should block mutation: decision=%q err=%v", decision, err)
}
}

func TestAdoptVerifiedRouteRestoresLocalState(t *testing.T) {
state, route := healthFixture(t, "direct", false)
current := findRoute(state.Inventory, route.ID)
current.Enabled = false
current.State = "deploying"
if err := state.Save(false); err != nil {
t.Fatal(err)
}

evidence := AuditEvidence{
Route: route.ID,
Status: "healthy",
Category: "in-sync",
ActualEgressIPv4: stringPointer(findServer(state.Inventory, route.ExitServer).Network.ExpectedEgressIPv4),
EgressMatchesDeclaredExit: true,
}
result, err := adoptVerifiedRoute(state, current, evidence, true, false)
if err != nil {
t.Fatal(err)
}
if result["state"] != "deployed" || result["enabled"] != true {
t.Fatalf("verified Route was not adopted: %#v", result)
}
reloaded, err := LoadState(state.PrivateDir)
if err != nil {
t.Fatal(err)
}
adopted := findRoute(reloaded.Inventory, route.ID)
if adopted == nil || adopted.State != "deployed" || !adopted.Enabled {
t.Fatalf("verified Route was not persisted locally: %#v", adopted)
}
}

Expand Down
17 changes: 12 additions & 5 deletions internal/steward/execution.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,17 @@ func deployRouteWithoutRender(ctx context.Context, state *State, routeID string)
return deployRoute(ctx, state, routeID, true, false)
}

func deploymentAuditAllowsMutation(route *Route, current AuditEvidence) error {
func deploymentAuditDecision(route *Route, current AuditEvidence) (string, error) {
if current.Category == "in-sync" {
return nil
if route.State == "deploying" || route.State == "deployed" {
return "adopt", nil
}
return "deploy", nil
}
if current.Category == "service-missing" && route.State != "deployed" {
return nil
return "deploy", nil
}
return errors.New("remote Route state is not safe to overwrite")
return "", errors.New("remote Route state is not safe to overwrite")
}

func markRouteDeploying(state *State, routeID string) error {
Expand All @@ -85,9 +88,13 @@ func deployRoute(ctx context.Context, state *State, routeID string, skipClientVa
}

current := AuditRoute(ctx, state, routeID)
if err := deploymentAuditAllowsMutation(route, current); err != nil {
decision, err := deploymentAuditDecision(route, current)
if err != nil {
return nil, &operationStageError{Stage: "remote-preflight-audit", StateChanged: "remote-state-unchanged", Retry: "audit", Err: err}
}
if decision == "adopt" {
return adoptVerifiedRoute(state, route, current, skipClientValidation, renderClients)
}
if err := markRouteDeploying(state, routeID); err != nil {
return nil, &operationStageError{Stage: "deployment-intent", StateChanged: "remote-state-unchanged", Retry: "deploy-route", Err: err}
}
Expand Down
2 changes: 1 addition & 1 deletion version.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
2.2.2
2.2.3