diff --git a/internal/bootstrap/teardown.go b/internal/bootstrap/teardown.go index b89dca3..03fe441 100644 --- a/internal/bootstrap/teardown.go +++ b/internal/bootstrap/teardown.go @@ -74,6 +74,9 @@ func (o *Orchestrator) Teardown(ctx context.Context, opts TeardownOptions) error } else if len(projects) > 0 { return fmt.Errorf("project environments still exist (%s): run 'urbex destroy ' for each of them first", strings.Join(projects, ", ")) } + if err := o.forgetStaleLedger(ctx, opts.GitOpsRepoPath); err != nil { + return err + } tfDir := filepath.Join(opts.GitOpsRepoPath, "terraform", "base") var repos []string @@ -157,7 +160,7 @@ func (o *Orchestrator) Teardown(ctx context.Context, opts TeardownOptions) error } // remainingProjects lists the project environments the GitOps repo still -// has: folders under environments/, and LXCs in the allocation ledger. +// has: folders under environments/. func remainingProjects(gitopsRepoPath string) ([]string, error) { found := map[string]bool{} envs, err := os.ReadDir(filepath.Join(gitopsRepoPath, "environments")) @@ -178,13 +181,6 @@ func remainingProjects(gitopsRepoPath string) ([]string, error) { } } } - ledger, err := allocation.Load(filepath.Join(gitopsRepoPath, "state", "allocations.json"), 0, 0) - if err != nil { - return nil, err - } - for _, hostname := range ledger.Hostnames() { - found[hostname] = true - } out := make([]string, 0, len(found)) for name := range found { out = append(out, name) @@ -201,3 +197,63 @@ func baseHostnames(cfg *platform.Config) []string { } return names } + +// forgetStaleLedger handles what the allocation ledger still holds once +// no project environment is left: entries an older urbex left behind +// (urbforge/urbex-cli#115). An entry whose machine is gone from Proxmox +// is removed, and said; one whose machine still exists stops the +// teardown, naming it - nothing is deleted that urbex doesn't account for. +// Without a way to list Proxmox's machines it names them all. +func (o *Orchestrator) forgetStaleLedger(ctx context.Context, gitopsRepoPath string) error { + path := filepath.Join(gitopsRepoPath, "state", "allocations.json") + ledger, err := allocation.Load(path, 0, 0) + if err != nil { + return err + } + hostnames := ledger.Hostnames() + if len(hostnames) == 0 { + return nil + } + sort.Strings(hostnames) + describe := func(hs []string) string { + var parts []string + for _, h := range hs { + e, _ := ledger.Lookup(h) + parts = append(parts, fmt.Sprintf("%s (VMID %d)", h, e.VMID)) + } + return strings.Join(parts, ", ") + } + lister, ok := o.Proxmox.(tfstate.ContainerLister) + if !ok { + return fmt.Errorf("the allocation ledger still has machines no project environment holds: %s. If they are gone from Proxmox, remove their entries from %s and run teardown again", describe(hostnames), path) + } + ids, err := lister.ListContainerIDs(ctx, o.Config.Proxmox.Node) + if err != nil { + return fmt.Errorf("listing proxmox containers: %w", err) + } + exists := make(map[int]bool, len(ids)) + for _, id := range ids { + exists[id] = true + } + var gone, live []string + for _, h := range hostnames { + if e, _ := ledger.Lookup(h); exists[e.VMID] { + live = append(live, h) + } else { + gone = append(gone, h) + } + } + if len(gone) > 0 { + fmt.Fprintf(o.Out, "Forgot stale entries of the allocation ledger, whose machines are gone from Proxmox: %s.\n", describe(gone)) + for _, h := range gone { + ledger.Remove(h) + } + if err := ledger.Save(path); err != nil { + return fmt.Errorf("saving allocation ledger: %w", err) + } + } + if len(live) > 0 { + return fmt.Errorf("machines no project environment holds still exist on Proxmox: %s. Delete them in pool %s (or put their application back in its urbex.yaml and run 'urbex destroy '), then run teardown again", describe(live), o.Config.Proxmox.Pool) + } + return nil +} diff --git a/internal/bootstrap/teardown_test.go b/internal/bootstrap/teardown_test.go index 4f89549..58b7afa 100644 --- a/internal/bootstrap/teardown_test.go +++ b/internal/bootstrap/teardown_test.go @@ -97,3 +97,50 @@ func TestTeardown_DestroysBaseServicesAndForgetsGeneratedCredentials(t *testing. t.Errorf("expected the gitops repo committed locally, last commit %q (%v)", out, err) } } + +// idProxmox also lists its machines' VMIDs, as proxmoxapi.Client does. +type idProxmox struct { + fakeProxmox + ids []int +} + +func (p *idProxmox) ListContainerIDs(context.Context, string) ([]int, error) { return p.ids, nil } + +// With no environment left, teardown forgets the ledger entries whose +// machines are gone from Proxmox, and stops on one whose machine still +// exists, naming it (urbforge/urbex-cli#115). +func TestForgetStaleLedger(t *testing.T) { + write := func(dir string) { + os.MkdirAll(filepath.Join(dir, "state"), 0o755) + os.WriteFile(filepath.Join(dir, "state", "allocations.json"), []byte(`{"entries":{"pgtest-board-staging":{"vmid":9129,"ip":"192.168.1.239"},"shop-api-prod":{"vmid":9130,"ip":"192.168.1.240"}},"nextVmid":9131,"nextIpOffset":241}`), 0o644) + } + for _, tc := range []struct { + name string + ids []int + wantErr string + left string + }{ + {"all gone", []int{9000, 9001}, "", ""}, + {"one still exists", []int{9130}, "shop-api-prod (VMID 9130)", "shop-api-prod"}, + } { + t.Run(tc.name, func(t *testing.T) { + o, _, _ := newRunOrchestrator(t, &fakeProxmox{}) + o.Proxmox = &idProxmox{ids: tc.ids} + var out bytes.Buffer + o.Out = &out + dir := t.TempDir() + write(dir) + err := o.forgetStaleLedger(context.Background(), dir) + if (err == nil) != (tc.wantErr == "") || (err != nil && !strings.Contains(err.Error(), tc.wantErr)) { + t.Fatalf("err = %v, want %q", err, tc.wantErr) + } + if !strings.Contains(out.String(), "pgtest-board-staging (VMID 9129)") { + t.Errorf("output should name the forgotten entry: %q", out.String()) + } + data, _ := os.ReadFile(filepath.Join(dir, "state", "allocations.json")) + if strings.Contains(string(data), "pgtest-board-staging") || (tc.left != "" && !strings.Contains(string(data), tc.left)) { + t.Errorf("ledger after: %s", data) + } + }) + } +} diff --git a/internal/provision/ledger.go b/internal/provision/ledger.go new file mode 100644 index 0000000..ced8aa1 --- /dev/null +++ b/internal/provision/ledger.go @@ -0,0 +1,88 @@ +package provision + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/urbforge/urbex-cli/internal/tfstate" +) + +// stateHostnames are the hostnames of the machines in the environment's +// Terraform state (module.service[""], module.vm[""]): +// what exists, whether urbex.yaml still declares it or not. +func stateHostnames(tfDir string) (map[string]bool, error) { + found := map[string]bool{} + data, err := os.ReadFile(filepath.Join(tfDir, "terraform.tfstate")) + if os.IsNotExist(err) { + return found, nil + } + if err != nil { + return nil, err + } + containers, err := tfstate.Containers(data) + if err != nil { + return nil, err + } + for _, c := range containers { + if h := hostnameOf(c.Address); h != "" { + found[h] = true + } + } + return found, nil +} + +// hostnameOf is the key of a resource address: module.service["x"].... → x. +func hostnameOf(address string) string { + i := strings.Index(address, `["`) + if i < 0 { + return "" + } + rest := address[i+2:] + j := strings.Index(rest, `"]`) + if j < 0 { + return "" + } + return rest[:j] +} + +// freeGone removes from the allocation ledger the machines that were in +// the environment's Terraform state (before) and no longer are (after), +// and that urbex.yaml doesn't declare (keep): an application removed from +// urbex.yaml, deleted by apply (urbforge/urbex-cli#115). Their addresses +// are free again; urbex teardown would otherwise take the entries for a +// live environment. +func (o *Orchestrator) freeGone(opts Options, before, after, keep map[string]bool) error { + var gone []string + for h := range before { + if !after[h] && !keep[h] { + gone = append(gone, h) + } + } + if len(gone) == 0 { + return nil + } + sort.Strings(gone) + ledgerPath := filepath.Join(opts.GitOpsRepoPath, "state", "allocations.json") + ledger, err := o.loadLedger(opts) + if err != nil { + return err + } + var freed []string + for _, h := range gone { + if _, ok := ledger.Lookup(h); ok { + ledger.Remove(h) + freed = append(freed, h) + } + } + if len(freed) == 0 { + return nil + } + if err := ledger.Save(ledgerPath); err != nil { + return fmt.Errorf("saving allocation ledger: %w", err) + } + fmt.Fprintf(o.Out, "Freed the addresses of %s: no longer in urbex.yaml, and deleted.\n", strings.Join(freed, ", ")) + return nil +} diff --git a/internal/provision/ledger_test.go b/internal/provision/ledger_test.go new file mode 100644 index 0000000..9168b6a --- /dev/null +++ b/internal/provision/ledger_test.go @@ -0,0 +1,69 @@ +package provision + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/urbforge/urbex-cli/internal/allocation" +) + +func writeState(t *testing.T, dir string, hostnames ...string) { + t.Helper() + var res []string + for i, h := range hostnames { + res = append(res, `{"module":"module.service[\"`+h+`\"]","mode":"managed","type":"proxmox_virtual_environment_container","name":"this","instances":[{"attributes":{"vm_id":`+string(rune('0'+i))+`}}]}`) + } + data := `{"version":4,"resources":[` + strings.Join(res, ",") + `]}` + if err := os.WriteFile(filepath.Join(dir, "terraform.tfstate"), []byte(data), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestStateHostnames(t *testing.T) { + dir := t.TempDir() + if got, err := stateHostnames(dir); err != nil || len(got) != 0 { + t.Fatalf("no state: %v, %v", got, err) + } + writeState(t, dir, "shop-api-prod", "shop-board-prod") + got, err := stateHostnames(dir) + if err != nil || !got["shop-api-prod"] || !got["shop-board-prod"] || len(got) != 2 { + t.Errorf("got %v, %v", got, err) + } +} + +// An application removed from urbex.yaml and deleted by apply frees its +// ledger entry; one still declared keeps it, and so does what stays in the +// state (urbforge/urbex-cli#115). +func TestFreeGone(t *testing.T) { + repo := t.TempDir() + ledgerPath := filepath.Join(repo, "state", "allocations.json") + l, _ := allocation.Load(ledgerPath, 9100, 210) + for _, h := range []string{"shop-api-prod", "shop-board-prod", "shop-db-prod", "other-web-prod"} { + if _, err := l.Allocate(h, "192.168.1.0/24", "192.168.1.1"); err != nil { + t.Fatal(err) + } + } + if err := l.Save(ledgerPath); err != nil { + t.Fatal(err) + } + var out bytes.Buffer + o := &Orchestrator{Config: validPlatformConfig(), Out: &out} + before := map[string]bool{"shop-api-prod": true, "shop-board-prod": true, "shop-db-prod": true} + after := map[string]bool{"shop-api-prod": true} + keep := map[string]bool{"shop-api-prod": true, "shop-db-prod": true} // db: deleted outside, still declared + if err := o.freeGone(Options{GitOpsRepoPath: repo}, before, after, keep); err != nil { + t.Fatal(err) + } + l, _ = allocation.Load(ledgerPath, 9100, 210) + for h, want := range map[string]bool{"shop-api-prod": true, "shop-board-prod": false, "shop-db-prod": true, "other-web-prod": true} { + if _, ok := l.Lookup(h); ok != want { + t.Errorf("%s in the ledger: %v, want %v", h, ok, want) + } + } + if !strings.Contains(out.String(), "Freed the addresses of shop-board-prod") { + t.Errorf("output: %q", out.String()) + } +} diff --git a/internal/provision/orchestrator.go b/internal/provision/orchestrator.go index eaf7acc..5e2507a 100644 --- a/internal/provision/orchestrator.go +++ b/internal/provision/orchestrator.go @@ -211,6 +211,12 @@ func (o *Orchestrator) Apply(ctx context.Context, opts Options) error { if err := o.Runner.Run(ctx, runner.Command{Dir: tfDir, Name: "terraform", Args: []string{"init"}, Env: env}); err != nil { return fmt.Errorf("terraform init: %w", err) } + // What the state holds now, vanished machines included: those that + // leave it and urbex.yaml no longer declares free their addresses. + before, err := stateHostnames(tfDir) + if err != nil { + return fmt.Errorf("reading terraform state: %w", err) + } if err := o.forgetVanished(ctx, tfDir, env); err != nil { return err } @@ -233,6 +239,17 @@ func (o *Orchestrator) Apply(ctx context.Context, opts Options) error { if err := runner.TerraformChange(ctx, o.Runner, o.Out, tfDir, "apply", env); err != nil { return err } + after, err := stateHostnames(tfDir) + if err != nil { + return fmt.Errorf("reading terraform state: %w", err) + } + declared := map[string]bool{} + for _, svc := range o.Manifest.Services { + declared[Hostname(o.Manifest.Project, svc.Name, opts.Env)] = true + } + if err := o.freeGone(opts, before, after, declared); err != nil { + return err + } if o.cloneVMID > 0 { if err := runner.TerraformGrowClones(ctx, o.Runner, o.Out, tfDir, env); err != nil { return err @@ -1131,6 +1148,12 @@ func (o *Orchestrator) Destroy(ctx context.Context, opts Options) error { if err := o.Runner.Run(ctx, runner.Command{Dir: tfDir, Name: "terraform", Args: []string{"init"}, Env: env}); err != nil { return fmt.Errorf("terraform init: %w", err) } + // Every machine of the state goes, those urbex.yaml no longer declares + // too: their ledger entries go with them (urbforge/urbex-cli#115). + inState, err := stateHostnames(tfDir) + if err != nil { + return fmt.Errorf("reading terraform state: %w", err) + } if err := o.forgetVanished(ctx, tfDir, env); err != nil { return err } @@ -1198,6 +1221,9 @@ func (o *Orchestrator) Destroy(ctx context.Context, opts Options) error { for _, hostname := range hostnames { ledger.Remove(hostname) } + for hostname := range inState { + ledger.Remove(hostname) + } if err := ledger.Save(ledgerPath); err != nil { return fmt.Errorf("saving allocation ledger: %w", err) }