From a21f80b86e0243c14dc32137cd83f35272bfbbee Mon Sep 17 00:00:00 2001 From: Teg79 Date: Tue, 17 Nov 2026 19:02:21 +0100 Subject: [PATCH] fix(allocation): destroy and apply free the ledger entries of machines gone; teardown forgets stale ones apply frees the entries of machines that left the Terraform state and urbex.yaml no longer declares; destroy frees every machine the state held, not only the declared ones. teardown, with no environment left, forgets entries whose machines are gone from Proxmox and stops, naming them, on those that still exist. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182yxsfwRowwnxCfgrQwjnA --- internal/bootstrap/teardown.go | 72 ++++++++++++++++++++--- internal/bootstrap/teardown_test.go | 47 +++++++++++++++ internal/provision/ledger.go | 88 +++++++++++++++++++++++++++++ internal/provision/ledger_test.go | 69 ++++++++++++++++++++++ internal/provision/orchestrator.go | 26 +++++++++ 5 files changed, 294 insertions(+), 8 deletions(-) create mode 100644 internal/provision/ledger.go create mode 100644 internal/provision/ledger_test.go diff --git a/internal/bootstrap/teardown.go b/internal/bootstrap/teardown.go index b89dca3..03fe441 100644 --- a/internal/bootstrap/teardown.go +++ b/internal/bootstrap/teardown.go @@ -74,6 +74,9 @@ func (o *Orchestrator) Teardown(ctx context.Context, opts TeardownOptions) error } else if len(projects) > 0 { return fmt.Errorf("project environments still exist (%s): run 'urbex destroy ' for each of them first", strings.Join(projects, ", ")) } + if err := o.forgetStaleLedger(ctx, opts.GitOpsRepoPath); err != nil { + return err + } tfDir := filepath.Join(opts.GitOpsRepoPath, "terraform", "base") var repos []string @@ -157,7 +160,7 @@ func (o *Orchestrator) Teardown(ctx context.Context, opts TeardownOptions) error } // remainingProjects lists the project environments the GitOps repo still -// has: folders under environments/, and LXCs in the allocation ledger. +// has: folders under environments/. func remainingProjects(gitopsRepoPath string) ([]string, error) { found := map[string]bool{} envs, err := os.ReadDir(filepath.Join(gitopsRepoPath, "environments")) @@ -178,13 +181,6 @@ func remainingProjects(gitopsRepoPath string) ([]string, error) { } } } - ledger, err := allocation.Load(filepath.Join(gitopsRepoPath, "state", "allocations.json"), 0, 0) - if err != nil { - return nil, err - } - for _, hostname := range ledger.Hostnames() { - found[hostname] = true - } out := make([]string, 0, len(found)) for name := range found { out = append(out, name) @@ -201,3 +197,63 @@ func baseHostnames(cfg *platform.Config) []string { } return names } + +// forgetStaleLedger handles what the allocation ledger still holds once +// no project environment is left: entries an older urbex left behind +// (urbforge/urbex-cli#115). An entry whose machine is gone from Proxmox +// is removed, and said; one whose machine still exists stops the +// teardown, naming it - nothing is deleted that urbex doesn't account for. +// Without a way to list Proxmox's machines it names them all. +func (o *Orchestrator) forgetStaleLedger(ctx context.Context, gitopsRepoPath string) error { + path := filepath.Join(gitopsRepoPath, "state", "allocations.json") + ledger, err := allocation.Load(path, 0, 0) + if err != nil { + return err + } + hostnames := ledger.Hostnames() + if len(hostnames) == 0 { + return nil + } + sort.Strings(hostnames) + describe := func(hs []string) string { + var parts []string + for _, h := range hs { + e, _ := ledger.Lookup(h) + parts = append(parts, fmt.Sprintf("%s (VMID %d)", h, e.VMID)) + } + return strings.Join(parts, ", ") + } + lister, ok := o.Proxmox.(tfstate.ContainerLister) + if !ok { + return fmt.Errorf("the allocation ledger still has machines no project environment holds: %s. If they are gone from Proxmox, remove their entries from %s and run teardown again", describe(hostnames), path) + } + ids, err := lister.ListContainerIDs(ctx, o.Config.Proxmox.Node) + if err != nil { + return fmt.Errorf("listing proxmox containers: %w", err) + } + exists := make(map[int]bool, len(ids)) + for _, id := range ids { + exists[id] = true + } + var gone, live []string + for _, h := range hostnames { + if e, _ := ledger.Lookup(h); exists[e.VMID] { + live = append(live, h) + } else { + gone = append(gone, h) + } + } + if len(gone) > 0 { + fmt.Fprintf(o.Out, "Forgot stale entries of the allocation ledger, whose machines are gone from Proxmox: %s.\n", describe(gone)) + for _, h := range gone { + ledger.Remove(h) + } + if err := ledger.Save(path); err != nil { + return fmt.Errorf("saving allocation ledger: %w", err) + } + } + if len(live) > 0 { + return fmt.Errorf("machines no project environment holds still exist on Proxmox: %s. Delete them in pool %s (or put their application back in its urbex.yaml and run 'urbex destroy '), then run teardown again", describe(live), o.Config.Proxmox.Pool) + } + return nil +} diff --git a/internal/bootstrap/teardown_test.go b/internal/bootstrap/teardown_test.go index 4f89549..58b7afa 100644 --- a/internal/bootstrap/teardown_test.go +++ b/internal/bootstrap/teardown_test.go @@ -97,3 +97,50 @@ func TestTeardown_DestroysBaseServicesAndForgetsGeneratedCredentials(t *testing. t.Errorf("expected the gitops repo committed locally, last commit %q (%v)", out, err) } } + +// idProxmox also lists its machines' VMIDs, as proxmoxapi.Client does. +type idProxmox struct { + fakeProxmox + ids []int +} + +func (p *idProxmox) ListContainerIDs(context.Context, string) ([]int, error) { return p.ids, nil } + +// With no environment left, teardown forgets the ledger entries whose +// machines are gone from Proxmox, and stops on one whose machine still +// exists, naming it (urbforge/urbex-cli#115). +func TestForgetStaleLedger(t *testing.T) { + write := func(dir string) { + os.MkdirAll(filepath.Join(dir, "state"), 0o755) + os.WriteFile(filepath.Join(dir, "state", "allocations.json"), []byte(`{"entries":{"pgtest-board-staging":{"vmid":9129,"ip":"192.168.1.239"},"shop-api-prod":{"vmid":9130,"ip":"192.168.1.240"}},"nextVmid":9131,"nextIpOffset":241}`), 0o644) + } + for _, tc := range []struct { + name string + ids []int + wantErr string + left string + }{ + {"all gone", []int{9000, 9001}, "", ""}, + {"one still exists", []int{9130}, "shop-api-prod (VMID 9130)", "shop-api-prod"}, + } { + t.Run(tc.name, func(t *testing.T) { + o, _, _ := newRunOrchestrator(t, &fakeProxmox{}) + o.Proxmox = &idProxmox{ids: tc.ids} + var out bytes.Buffer + o.Out = &out + dir := t.TempDir() + write(dir) + err := o.forgetStaleLedger(context.Background(), dir) + if (err == nil) != (tc.wantErr == "") || (err != nil && !strings.Contains(err.Error(), tc.wantErr)) { + t.Fatalf("err = %v, want %q", err, tc.wantErr) + } + if !strings.Contains(out.String(), "pgtest-board-staging (VMID 9129)") { + t.Errorf("output should name the forgotten entry: %q", out.String()) + } + data, _ := os.ReadFile(filepath.Join(dir, "state", "allocations.json")) + if strings.Contains(string(data), "pgtest-board-staging") || (tc.left != "" && !strings.Contains(string(data), tc.left)) { + t.Errorf("ledger after: %s", data) + } + }) + } +} diff --git a/internal/provision/ledger.go b/internal/provision/ledger.go new file mode 100644 index 0000000..ced8aa1 --- /dev/null +++ b/internal/provision/ledger.go @@ -0,0 +1,88 @@ +package provision + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/urbforge/urbex-cli/internal/tfstate" +) + +// stateHostnames are the hostnames of the machines in the environment's +// Terraform state (module.service[""], module.vm[""]): +// what exists, whether urbex.yaml still declares it or not. +func stateHostnames(tfDir string) (map[string]bool, error) { + found := map[string]bool{} + data, err := os.ReadFile(filepath.Join(tfDir, "terraform.tfstate")) + if os.IsNotExist(err) { + return found, nil + } + if err != nil { + return nil, err + } + containers, err := tfstate.Containers(data) + if err != nil { + return nil, err + } + for _, c := range containers { + if h := hostnameOf(c.Address); h != "" { + found[h] = true + } + } + return found, nil +} + +// hostnameOf is the key of a resource address: module.service["x"].... → x. +func hostnameOf(address string) string { + i := strings.Index(address, `["`) + if i < 0 { + return "" + } + rest := address[i+2:] + j := strings.Index(rest, `"]`) + if j < 0 { + return "" + } + return rest[:j] +} + +// freeGone removes from the allocation ledger the machines that were in +// the environment's Terraform state (before) and no longer are (after), +// and that urbex.yaml doesn't declare (keep): an application removed from +// urbex.yaml, deleted by apply (urbforge/urbex-cli#115). Their addresses +// are free again; urbex teardown would otherwise take the entries for a +// live environment. +func (o *Orchestrator) freeGone(opts Options, before, after, keep map[string]bool) error { + var gone []string + for h := range before { + if !after[h] && !keep[h] { + gone = append(gone, h) + } + } + if len(gone) == 0 { + return nil + } + sort.Strings(gone) + ledgerPath := filepath.Join(opts.GitOpsRepoPath, "state", "allocations.json") + ledger, err := o.loadLedger(opts) + if err != nil { + return err + } + var freed []string + for _, h := range gone { + if _, ok := ledger.Lookup(h); ok { + ledger.Remove(h) + freed = append(freed, h) + } + } + if len(freed) == 0 { + return nil + } + if err := ledger.Save(ledgerPath); err != nil { + return fmt.Errorf("saving allocation ledger: %w", err) + } + fmt.Fprintf(o.Out, "Freed the addresses of %s: no longer in urbex.yaml, and deleted.\n", strings.Join(freed, ", ")) + return nil +} diff --git a/internal/provision/ledger_test.go b/internal/provision/ledger_test.go new file mode 100644 index 0000000..9168b6a --- /dev/null +++ b/internal/provision/ledger_test.go @@ -0,0 +1,69 @@ +package provision + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/urbforge/urbex-cli/internal/allocation" +) + +func writeState(t *testing.T, dir string, hostnames ...string) { + t.Helper() + var res []string + for i, h := range hostnames { + res = append(res, `{"module":"module.service[\"`+h+`\"]","mode":"managed","type":"proxmox_virtual_environment_container","name":"this","instances":[{"attributes":{"vm_id":`+string(rune('0'+i))+`}}]}`) + } + data := `{"version":4,"resources":[` + strings.Join(res, ",") + `]}` + if err := os.WriteFile(filepath.Join(dir, "terraform.tfstate"), []byte(data), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestStateHostnames(t *testing.T) { + dir := t.TempDir() + if got, err := stateHostnames(dir); err != nil || len(got) != 0 { + t.Fatalf("no state: %v, %v", got, err) + } + writeState(t, dir, "shop-api-prod", "shop-board-prod") + got, err := stateHostnames(dir) + if err != nil || !got["shop-api-prod"] || !got["shop-board-prod"] || len(got) != 2 { + t.Errorf("got %v, %v", got, err) + } +} + +// An application removed from urbex.yaml and deleted by apply frees its +// ledger entry; one still declared keeps it, and so does what stays in the +// state (urbforge/urbex-cli#115). +func TestFreeGone(t *testing.T) { + repo := t.TempDir() + ledgerPath := filepath.Join(repo, "state", "allocations.json") + l, _ := allocation.Load(ledgerPath, 9100, 210) + for _, h := range []string{"shop-api-prod", "shop-board-prod", "shop-db-prod", "other-web-prod"} { + if _, err := l.Allocate(h, "192.168.1.0/24", "192.168.1.1"); err != nil { + t.Fatal(err) + } + } + if err := l.Save(ledgerPath); err != nil { + t.Fatal(err) + } + var out bytes.Buffer + o := &Orchestrator{Config: validPlatformConfig(), Out: &out} + before := map[string]bool{"shop-api-prod": true, "shop-board-prod": true, "shop-db-prod": true} + after := map[string]bool{"shop-api-prod": true} + keep := map[string]bool{"shop-api-prod": true, "shop-db-prod": true} // db: deleted outside, still declared + if err := o.freeGone(Options{GitOpsRepoPath: repo}, before, after, keep); err != nil { + t.Fatal(err) + } + l, _ = allocation.Load(ledgerPath, 9100, 210) + for h, want := range map[string]bool{"shop-api-prod": true, "shop-board-prod": false, "shop-db-prod": true, "other-web-prod": true} { + if _, ok := l.Lookup(h); ok != want { + t.Errorf("%s in the ledger: %v, want %v", h, ok, want) + } + } + if !strings.Contains(out.String(), "Freed the addresses of shop-board-prod") { + t.Errorf("output: %q", out.String()) + } +} diff --git a/internal/provision/orchestrator.go b/internal/provision/orchestrator.go index eaf7acc..5e2507a 100644 --- a/internal/provision/orchestrator.go +++ b/internal/provision/orchestrator.go @@ -211,6 +211,12 @@ func (o *Orchestrator) Apply(ctx context.Context, opts Options) error { if err := o.Runner.Run(ctx, runner.Command{Dir: tfDir, Name: "terraform", Args: []string{"init"}, Env: env}); err != nil { return fmt.Errorf("terraform init: %w", err) } + // What the state holds now, vanished machines included: those that + // leave it and urbex.yaml no longer declares free their addresses. + before, err := stateHostnames(tfDir) + if err != nil { + return fmt.Errorf("reading terraform state: %w", err) + } if err := o.forgetVanished(ctx, tfDir, env); err != nil { return err } @@ -233,6 +239,17 @@ func (o *Orchestrator) Apply(ctx context.Context, opts Options) error { if err := runner.TerraformChange(ctx, o.Runner, o.Out, tfDir, "apply", env); err != nil { return err } + after, err := stateHostnames(tfDir) + if err != nil { + return fmt.Errorf("reading terraform state: %w", err) + } + declared := map[string]bool{} + for _, svc := range o.Manifest.Services { + declared[Hostname(o.Manifest.Project, svc.Name, opts.Env)] = true + } + if err := o.freeGone(opts, before, after, declared); err != nil { + return err + } if o.cloneVMID > 0 { if err := runner.TerraformGrowClones(ctx, o.Runner, o.Out, tfDir, env); err != nil { return err @@ -1131,6 +1148,12 @@ func (o *Orchestrator) Destroy(ctx context.Context, opts Options) error { if err := o.Runner.Run(ctx, runner.Command{Dir: tfDir, Name: "terraform", Args: []string{"init"}, Env: env}); err != nil { return fmt.Errorf("terraform init: %w", err) } + // Every machine of the state goes, those urbex.yaml no longer declares + // too: their ledger entries go with them (urbforge/urbex-cli#115). + inState, err := stateHostnames(tfDir) + if err != nil { + return fmt.Errorf("reading terraform state: %w", err) + } if err := o.forgetVanished(ctx, tfDir, env); err != nil { return err } @@ -1198,6 +1221,9 @@ func (o *Orchestrator) Destroy(ctx context.Context, opts Options) error { for _, hostname := range hostnames { ledger.Remove(hostname) } + for hostname := range inState { + ledger.Remove(hostname) + } if err := ledger.Save(ledgerPath); err != nil { return fmt.Errorf("saving allocation ledger: %w", err) }