Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 64 additions & 8 deletions internal/bootstrap/teardown.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ func (o *Orchestrator) Teardown(ctx context.Context, opts TeardownOptions) error
} else if len(projects) > 0 {
return fmt.Errorf("project environments still exist (%s): run 'urbex destroy <env>' for each of them first", strings.Join(projects, ", "))
}
if err := o.forgetStaleLedger(ctx, opts.GitOpsRepoPath); err != nil {
return err
}

tfDir := filepath.Join(opts.GitOpsRepoPath, "terraform", "base")
var repos []string
Expand Down Expand Up @@ -157,7 +160,7 @@ func (o *Orchestrator) Teardown(ctx context.Context, opts TeardownOptions) error
}

// remainingProjects lists the project environments the GitOps repo still
// has: folders under environments/, and LXCs in the allocation ledger.
// has: folders under environments/.
func remainingProjects(gitopsRepoPath string) ([]string, error) {
found := map[string]bool{}
envs, err := os.ReadDir(filepath.Join(gitopsRepoPath, "environments"))
Expand All @@ -178,13 +181,6 @@ func remainingProjects(gitopsRepoPath string) ([]string, error) {
}
}
}
ledger, err := allocation.Load(filepath.Join(gitopsRepoPath, "state", "allocations.json"), 0, 0)
if err != nil {
return nil, err
}
for _, hostname := range ledger.Hostnames() {
found[hostname] = true
}
out := make([]string, 0, len(found))
for name := range found {
out = append(out, name)
Expand All @@ -201,3 +197,63 @@ func baseHostnames(cfg *platform.Config) []string {
}
return names
}

// forgetStaleLedger handles what the allocation ledger still holds once
// no project environment is left: entries an older urbex left behind
// (urbforge/urbex-cli#115). An entry whose machine is gone from Proxmox
// is removed, and said; one whose machine still exists stops the
// teardown, naming it - nothing is deleted that urbex doesn't account for.
// Without a way to list Proxmox's machines it names them all.
func (o *Orchestrator) forgetStaleLedger(ctx context.Context, gitopsRepoPath string) error {
path := filepath.Join(gitopsRepoPath, "state", "allocations.json")
ledger, err := allocation.Load(path, 0, 0)
if err != nil {
return err
}
hostnames := ledger.Hostnames()
if len(hostnames) == 0 {
return nil
}
sort.Strings(hostnames)
describe := func(hs []string) string {
var parts []string
for _, h := range hs {
e, _ := ledger.Lookup(h)
parts = append(parts, fmt.Sprintf("%s (VMID %d)", h, e.VMID))
}
return strings.Join(parts, ", ")
}
lister, ok := o.Proxmox.(tfstate.ContainerLister)
if !ok {
return fmt.Errorf("the allocation ledger still has machines no project environment holds: %s. If they are gone from Proxmox, remove their entries from %s and run teardown again", describe(hostnames), path)
}
ids, err := lister.ListContainerIDs(ctx, o.Config.Proxmox.Node)
if err != nil {
return fmt.Errorf("listing proxmox containers: %w", err)
}
exists := make(map[int]bool, len(ids))
for _, id := range ids {
exists[id] = true
}
var gone, live []string
for _, h := range hostnames {
if e, _ := ledger.Lookup(h); exists[e.VMID] {
live = append(live, h)
} else {
gone = append(gone, h)
}
}
if len(gone) > 0 {
fmt.Fprintf(o.Out, "Forgot stale entries of the allocation ledger, whose machines are gone from Proxmox: %s.\n", describe(gone))
for _, h := range gone {
ledger.Remove(h)
}
if err := ledger.Save(path); err != nil {
return fmt.Errorf("saving allocation ledger: %w", err)
}
}
if len(live) > 0 {
return fmt.Errorf("machines no project environment holds still exist on Proxmox: %s. Delete them in pool %s (or put their application back in its urbex.yaml and run 'urbex destroy <env>'), then run teardown again", describe(live), o.Config.Proxmox.Pool)
}
return nil
}
47 changes: 47 additions & 0 deletions internal/bootstrap/teardown_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,50 @@ func TestTeardown_DestroysBaseServicesAndForgetsGeneratedCredentials(t *testing.
t.Errorf("expected the gitops repo committed locally, last commit %q (%v)", out, err)
}
}

// idProxmox also lists its machines' VMIDs, as proxmoxapi.Client does.
type idProxmox struct {
fakeProxmox
ids []int
}

func (p *idProxmox) ListContainerIDs(context.Context, string) ([]int, error) { return p.ids, nil }

// With no environment left, teardown forgets the ledger entries whose
// machines are gone from Proxmox, and stops on one whose machine still
// exists, naming it (urbforge/urbex-cli#115).
func TestForgetStaleLedger(t *testing.T) {
write := func(dir string) {
os.MkdirAll(filepath.Join(dir, "state"), 0o755)
os.WriteFile(filepath.Join(dir, "state", "allocations.json"), []byte(`{"entries":{"pgtest-board-staging":{"vmid":9129,"ip":"192.168.1.239"},"shop-api-prod":{"vmid":9130,"ip":"192.168.1.240"}},"nextVmid":9131,"nextIpOffset":241}`), 0o644)
}
for _, tc := range []struct {
name string
ids []int
wantErr string
left string
}{
{"all gone", []int{9000, 9001}, "", ""},
{"one still exists", []int{9130}, "shop-api-prod (VMID 9130)", "shop-api-prod"},
} {
t.Run(tc.name, func(t *testing.T) {
o, _, _ := newRunOrchestrator(t, &fakeProxmox{})
o.Proxmox = &idProxmox{ids: tc.ids}
var out bytes.Buffer
o.Out = &out
dir := t.TempDir()
write(dir)
err := o.forgetStaleLedger(context.Background(), dir)
if (err == nil) != (tc.wantErr == "") || (err != nil && !strings.Contains(err.Error(), tc.wantErr)) {
t.Fatalf("err = %v, want %q", err, tc.wantErr)
}
if !strings.Contains(out.String(), "pgtest-board-staging (VMID 9129)") {
t.Errorf("output should name the forgotten entry: %q", out.String())
}
data, _ := os.ReadFile(filepath.Join(dir, "state", "allocations.json"))
if strings.Contains(string(data), "pgtest-board-staging") || (tc.left != "" && !strings.Contains(string(data), tc.left)) {
t.Errorf("ledger after: %s", data)
}
})
}
}
88 changes: 88 additions & 0 deletions internal/provision/ledger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package provision

import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"

"github.com/urbforge/urbex-cli/internal/tfstate"
)

// stateHostnames are the hostnames of the machines in the environment's
// Terraform state (module.service["<hostname>"], module.vm["<hostname>"]):
// what exists, whether urbex.yaml still declares it or not.
func stateHostnames(tfDir string) (map[string]bool, error) {
found := map[string]bool{}
data, err := os.ReadFile(filepath.Join(tfDir, "terraform.tfstate"))
if os.IsNotExist(err) {
return found, nil
}
if err != nil {
return nil, err
}
containers, err := tfstate.Containers(data)
if err != nil {
return nil, err
}
for _, c := range containers {
if h := hostnameOf(c.Address); h != "" {
found[h] = true
}
}
return found, nil
}

// hostnameOf is the key of a resource address: module.service["x"].... → x.
func hostnameOf(address string) string {
i := strings.Index(address, `["`)
if i < 0 {
return ""
}
rest := address[i+2:]
j := strings.Index(rest, `"]`)
if j < 0 {
return ""
}
return rest[:j]
}

// freeGone removes from the allocation ledger the machines that were in
// the environment's Terraform state (before) and no longer are (after),
// and that urbex.yaml doesn't declare (keep): an application removed from
// urbex.yaml, deleted by apply (urbforge/urbex-cli#115). Their addresses
// are free again; urbex teardown would otherwise take the entries for a
// live environment.
func (o *Orchestrator) freeGone(opts Options, before, after, keep map[string]bool) error {
var gone []string
for h := range before {
if !after[h] && !keep[h] {
gone = append(gone, h)
}
}
if len(gone) == 0 {
return nil
}
sort.Strings(gone)
ledgerPath := filepath.Join(opts.GitOpsRepoPath, "state", "allocations.json")
ledger, err := o.loadLedger(opts)
if err != nil {
return err
}
var freed []string
for _, h := range gone {
if _, ok := ledger.Lookup(h); ok {
ledger.Remove(h)
freed = append(freed, h)
}
}
if len(freed) == 0 {
return nil
}
if err := ledger.Save(ledgerPath); err != nil {
return fmt.Errorf("saving allocation ledger: %w", err)
}
fmt.Fprintf(o.Out, "Freed the addresses of %s: no longer in urbex.yaml, and deleted.\n", strings.Join(freed, ", "))
return nil
}
69 changes: 69 additions & 0 deletions internal/provision/ledger_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package provision

import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"

"github.com/urbforge/urbex-cli/internal/allocation"
)

func writeState(t *testing.T, dir string, hostnames ...string) {
t.Helper()
var res []string
for i, h := range hostnames {
res = append(res, `{"module":"module.service[\"`+h+`\"]","mode":"managed","type":"proxmox_virtual_environment_container","name":"this","instances":[{"attributes":{"vm_id":`+string(rune('0'+i))+`}}]}`)
}
data := `{"version":4,"resources":[` + strings.Join(res, ",") + `]}`
if err := os.WriteFile(filepath.Join(dir, "terraform.tfstate"), []byte(data), 0o644); err != nil {
t.Fatal(err)
}
}

func TestStateHostnames(t *testing.T) {
dir := t.TempDir()
if got, err := stateHostnames(dir); err != nil || len(got) != 0 {
t.Fatalf("no state: %v, %v", got, err)
}
writeState(t, dir, "shop-api-prod", "shop-board-prod")
got, err := stateHostnames(dir)
if err != nil || !got["shop-api-prod"] || !got["shop-board-prod"] || len(got) != 2 {
t.Errorf("got %v, %v", got, err)
}
}

// An application removed from urbex.yaml and deleted by apply frees its
// ledger entry; one still declared keeps it, and so does what stays in the
// state (urbforge/urbex-cli#115).
func TestFreeGone(t *testing.T) {
repo := t.TempDir()
ledgerPath := filepath.Join(repo, "state", "allocations.json")
l, _ := allocation.Load(ledgerPath, 9100, 210)
for _, h := range []string{"shop-api-prod", "shop-board-prod", "shop-db-prod", "other-web-prod"} {
if _, err := l.Allocate(h, "192.168.1.0/24", "192.168.1.1"); err != nil {
t.Fatal(err)
}
}
if err := l.Save(ledgerPath); err != nil {
t.Fatal(err)
}
var out bytes.Buffer
o := &Orchestrator{Config: validPlatformConfig(), Out: &out}
before := map[string]bool{"shop-api-prod": true, "shop-board-prod": true, "shop-db-prod": true}
after := map[string]bool{"shop-api-prod": true}
keep := map[string]bool{"shop-api-prod": true, "shop-db-prod": true} // db: deleted outside, still declared
if err := o.freeGone(Options{GitOpsRepoPath: repo}, before, after, keep); err != nil {
t.Fatal(err)
}
l, _ = allocation.Load(ledgerPath, 9100, 210)
for h, want := range map[string]bool{"shop-api-prod": true, "shop-board-prod": false, "shop-db-prod": true, "other-web-prod": true} {
if _, ok := l.Lookup(h); ok != want {
t.Errorf("%s in the ledger: %v, want %v", h, ok, want)
}
}
if !strings.Contains(out.String(), "Freed the addresses of shop-board-prod") {
t.Errorf("output: %q", out.String())
}
}
26 changes: 26 additions & 0 deletions internal/provision/orchestrator.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,12 @@ func (o *Orchestrator) Apply(ctx context.Context, opts Options) error {
if err := o.Runner.Run(ctx, runner.Command{Dir: tfDir, Name: "terraform", Args: []string{"init"}, Env: env}); err != nil {
return fmt.Errorf("terraform init: %w", err)
}
// What the state holds now, vanished machines included: those that
// leave it and urbex.yaml no longer declares free their addresses.
before, err := stateHostnames(tfDir)
if err != nil {
return fmt.Errorf("reading terraform state: %w", err)
}
if err := o.forgetVanished(ctx, tfDir, env); err != nil {
return err
}
Expand All @@ -233,6 +239,17 @@ func (o *Orchestrator) Apply(ctx context.Context, opts Options) error {
if err := runner.TerraformChange(ctx, o.Runner, o.Out, tfDir, "apply", env); err != nil {
return err
}
after, err := stateHostnames(tfDir)
if err != nil {
return fmt.Errorf("reading terraform state: %w", err)
}
declared := map[string]bool{}
for _, svc := range o.Manifest.Services {
declared[Hostname(o.Manifest.Project, svc.Name, opts.Env)] = true
}
if err := o.freeGone(opts, before, after, declared); err != nil {
return err
}
if o.cloneVMID > 0 {
if err := runner.TerraformGrowClones(ctx, o.Runner, o.Out, tfDir, env); err != nil {
return err
Expand Down Expand Up @@ -1131,6 +1148,12 @@ func (o *Orchestrator) Destroy(ctx context.Context, opts Options) error {
if err := o.Runner.Run(ctx, runner.Command{Dir: tfDir, Name: "terraform", Args: []string{"init"}, Env: env}); err != nil {
return fmt.Errorf("terraform init: %w", err)
}
// Every machine of the state goes, those urbex.yaml no longer declares
// too: their ledger entries go with them (urbforge/urbex-cli#115).
inState, err := stateHostnames(tfDir)
if err != nil {
return fmt.Errorf("reading terraform state: %w", err)
}
if err := o.forgetVanished(ctx, tfDir, env); err != nil {
return err
}
Expand Down Expand Up @@ -1198,6 +1221,9 @@ func (o *Orchestrator) Destroy(ctx context.Context, opts Options) error {
for _, hostname := range hostnames {
ledger.Remove(hostname)
}
for hostname := range inState {
ledger.Remove(hostname)
}
if err := ledger.Save(ledgerPath); err != nil {
return fmt.Errorf("saving allocation ledger: %w", err)
}
Expand Down
Loading