From 7d613abf67c85f1beee24de73353bcff1a83de59 Mon Sep 17 00:00:00 2001 From: Edwin <18327273+thinkbig1979@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:57:51 +0200 Subject: [PATCH] fix(backup): capture capstan.db in every run and rewrite the DR runbook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit capstan.db was in no snapshot at all. The backup engine's scope was each stack's compose directory, tagged by stack ID; the database — user accounts, the encrypted git tokens and restic password, every setting, backup and auto-update policies, the stacks registry, update history and the audit log — was simply not backed up. Two failure modes compounded. The restic repository defaults to /restic-repo, the same volume the database lives on, so losing that one volume lost the database and the local snapshots together; the only surviving copy was whatever rclone had synced offsite, which contained stack directories and nothing else. And the runbook did not mention any of it, so an operator following it exactly got their compose files back and a Capstan with no users, no settings, no credentials, no policies and no history — discovered during an actual outage. WHAT CHANGES Every backup run, scheduled or manual, now snapshots capstan.db before the per-stack loop, under its own restic tag "capstan-database". Taking it first means a stack failure part-way through still leaves the database in the repository. The snapshot uses SQLite's VACUUM INTO (database.DB.VacuumInto), not a file copy. capstan.db runs in WAL mode, so a raw copy taken while writes are in flight can be torn or missing recent commits. VACUUM INTO observes one consistent snapshot and emits a standalone database with no -wal/-shm sidecars, which is what makes it safe to hand to restic as a single file. The staged copy is a full plaintext database and is removed in a defer, whatever happens next. A run that saves every stack but loses the database is reported "partial", never "success". Reporting green while the most important artifact is missing is the silent failure this issue exists to remove. The staging path is fixed at /backup-staging/capstan.db rather than a random temp dir, because it becomes the path recorded inside the snapshot and therefore the path an operator types during a restore. A random path would make the runbook unwritable. Exported as DatabaseSnapshotPath() and pinned by a test, since README names it literally. RUNBOOK README's disaster-recovery section is rewritten: database before stacks, stop the container before overwriting a live database, what STORAGE_KEY and JWT_SECRET are for, a four-point verification, and what is still recoverable when STORAGE_KEY is lost. Three corrections came out of actually executing it rather than writing it: - Steps 2-4 now run inside a throwaway container built from the image. restic and rclone ship in the image at pinned versions and a fresh recovery host usually has neither; the rclone remote path in Capstan's settings is a path as seen from inside the container, so running the documented `rclone sync` on the host fails with "directory not found" (observed). - The restore must delete capstan.db-wal / capstan.db-shm, which belong to the database being replaced. - The restored file must be chowned to appuser. The restore runs as root and Capstan does not, so a root-owned capstan.db leaves the server unable to write to its own database. Gates: go build, go vet clean; 674 tests in 9 packages (667 baseline + 7 added). Refs: agent-os-36o Co-Authored-By: Claude Opus 5 --- README.md | 173 +++++++++++++++++- backend/internal/database/database.go | 31 ++++ .../internal/database/database_vacuum_test.go | 72 ++++++++ backend/internal/services/backup.go | 150 +++++++++++++++ backend/internal/services/backup_test.go | 173 ++++++++++++++++++ 5 files changed, 591 insertions(+), 8 deletions(-) create mode 100644 backend/internal/database/database_vacuum_test.go diff --git a/README.md b/README.md index c4f9746..439d605 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,35 @@ Capstan includes a built-in backup engine powered by [restic](https://restic.net ### What gets backed up -Each stack's compose directory is backed up as a restic snapshot tagged with the stack ID. +Every backup run captures two kinds of thing: + +| Artifact | restic tag | Notes | +| --- | --- | --- | +| Each stack's compose directory | the stack ID | One snapshot per enabled stack | +| `capstan.db` | `capstan-database` | Captured on **every** run, before the stacks | + +The database is what makes an instance *this* instance: user accounts, the +encrypted git tokens and restic password, every setting, backup and auto-update +policies, the stacks registry, and the audit log. Without it a restore gives you +your compose files back and nothing else. + +It is snapshotted with SQLite's `VACUUM INTO` rather than copied. `capstan.db` +runs in WAL mode, so a plain file copy taken while writes are in flight can be +torn or missing recent commits. + +A run that saves every stack but fails to capture the database is reported as +**partial**, never success. + Backups can be triggered manually (Settings, Backup tab) or run on a schedule. +> **`STORAGE_KEY` is not in the backup, and that is deliberate.** The secrets +> inside `capstan.db` are encrypted with a key derived from it, so a stolen +> backup does not yield your git tokens. The corollary is that restoring onto a +> host with a *different* `STORAGE_KEY` gives you a database whose secrets +> cannot be decrypted. Store `STORAGE_KEY` and `JWT_SECRET` in the same password +> manager as the restic repository password. All three are needed for a complete +> recovery. + ### Configuration The quickest path is the **Settings UI** (Settings, Backup). All fields save to the @@ -157,15 +183,146 @@ curl -X POST http://localhost:5001/api/v1/backups/restore \ ### Disaster recovery -If the host is lost, recover from an rclone remote: +Follow this in order. **The database is restored before the stacks** — restoring +stacks first gives you a Capstan that does not know they exist. + +#### Before you start, you need + +| Item | Why | +| --- | --- | +| The restic repository password | Without it the repository cannot be opened at all | +| `STORAGE_KEY` | The old value, or every stored secret is unreadable | +| `JWT_SECRET` | A different value invalidates existing sessions (survivable) | +| Access to the rclone remote | The only copy that survives losing the data volume | + +If you have the repository but not `STORAGE_KEY`, you can still recover +everything except the encrypted secrets — see step 6. + +#### 1. Deploy a fresh instance, stopped + +Bring up a new Capstan with the same `./data` bind mount path and the **same** +`STORAGE_KEY` and `JWT_SECRET` as the lost instance, then stop the container: + +```bash +docker compose -f docker-compose.prod.yaml up -d +docker compose -f docker-compose.prod.yaml stop capstan +``` + +Stopping matters. Overwriting `capstan.db` underneath a running server corrupts +it — the server holds the WAL open and will write over what you just restored. + +#### Run steps 2–4 inside the image, not on the host + +Every command below is run in a throwaway container built from the Capstan +image, with the same volumes. Three reasons, all learned the hard way in a real +drill: + +- `restic` and `rclone` ship **inside** the image at pinned versions. A freshly + provisioned recovery host usually has neither, and installing them mid-outage + is a poor use of the outage. +- The rclone remote path stored in Capstan's settings is a path **as seen from + inside the container**. Running the same `rclone sync` on the host resolves it + against the host filesystem and fails with `directory not found`. +- The paths inside the restic snapshot are container paths, so restoring in the + same namespace keeps them meaningful. + +Define this once, adjusting the paths to your deployment: + +```bash +alias capstan-tools='docker run --rm \ + -v ./data:/app/data \ + -v ~/.config/rclone/rclone.conf:/home/appuser/.config/rclone/rclone.conf:ro \ + --entrypoint sh ghcr.io/thinkbig1979/capstan:latest -c' +``` + +#### 2. Get the repository back from the remote + +```bash +capstan-tools 'rclone sync : /app/data/restic-repo --stats-one-line' +``` + +`:` are exactly the values shown in Settings → Backup. This is +what the UI's DR Restore does; doing it by hand keeps step 1's "container +stopped" rule intact. + +#### 3. Find the database snapshot + +```bash +capstan-tools ' + printf "%s" "$RESTIC_PASSWORD" > /tmp/pw + export RESTIC_REPOSITORY=/app/data/restic-repo RESTIC_PASSWORD_FILE=/tmp/pw + restic snapshots --tag capstan-database' +``` + +The newest one is normally what you want. Note its short ID. + +#### 4. Restore the database + +The snapshot contains the file at `/app/data/backup-staging/capstan.db`, so a +restore reproduces that path underneath `--target`: + +```bash +capstan-tools ' + set -e + printf "%s" "$RESTIC_PASSWORD" > /tmp/pw + export RESTIC_REPOSITORY=/app/data/restic-repo RESTIC_PASSWORD_FILE=/tmp/pw + restic restore --target /tmp/capstan-restore + cp /tmp/capstan-restore/app/data/backup-staging/capstan.db /app/data/capstan.db + rm -rf /tmp/capstan-restore + rm -f /app/data/capstan.db-wal /app/data/capstan.db-shm + chown appuser:appuser /app/data/capstan.db' +``` + +Two details that will bite if skipped: + +- **Delete `capstan.db-wal` and `capstan.db-shm`.** They belong to the database + you just replaced. Left in place, SQLite tries to apply them to the restored + file. +- **`chown appuser:appuser`.** The restore runs as root; Capstan runs as + `appuser` (UID 1000 by default, remappable via `PUID`/`PGID`). A root-owned + `capstan.db` leaves the server unable to write to its own database. + +#### 5. Start Capstan and verify before going further + +```bash +docker compose -f docker-compose.prod.yaml start capstan +``` + +Check all four, in this order. Each one fails differently: + +1. **Log in** with a pre-existing account. Failure here means the wrong + `capstan.db`, not a `STORAGE_KEY` problem. +2. **Settings → Backup** shows your repository path, retention and schedule. + Empty defaults mean the database did not actually land. +3. **A git-backed stack can pull.** This is the `STORAGE_KEY` test — the token + is decrypted to run it. An authentication failure here with the right + password means the key differs from the one that encrypted it. +4. **Settings → Backup → History** lists runs from before the incident, and your + backup policies are still enabled. + +#### 6. If `STORAGE_KEY` was lost + +Everything except the encrypted secrets is recoverable. Accounts, settings, +policies, history and the stacks registry all come back. What must be re-entered +by hand: the restic repository password, git HTTPS tokens, and any other stored +credential. Re-enter them in Settings; the new values are encrypted with the new +key. + +#### 7. Restore the stacks + +Only now, with a working Capstan that knows about them, restore the compose +directories from Settings → Backup → Snapshots, or via the API: + +```bash +curl -X POST http://localhost:5001/api/v1/backups/restore \ + -H 'Content-Type: application/json' \ + -d '{"stackId":"","snapshotId":"","confirm":true}' +``` -1. Deploy a fresh Capstan instance with the same `./data` bind mount path. -2. Trigger a DR restore (Settings, Backup, DR Restore). This syncs the full - restic repository from the configured remote back to `/app/data/restic-repo`. -3. Restore individual stacks via the Snapshots panel. +#### Practise this -The restic repository password is required for DR recovery. Store it separately -from the server (e.g. a password manager). +A runbook that has never been executed is a hypothesis. Run it end to end +against a scratch instance at least once, before you need it. ## Volume Path Identity diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go index ae34753..4aeff60 100644 --- a/backend/internal/database/database.go +++ b/backend/internal/database/database.go @@ -2,7 +2,9 @@ package database import ( "database/sql" + "fmt" "os" + "strings" "time" _ "modernc.org/sqlite" @@ -92,3 +94,32 @@ func NewWithMigrationsAndEncryptor(dataDir string, encryptor TokenEncryptor) (*D func (d *DB) Close() error { return d.db.Close() } + +// VacuumInto writes a consistent point-in-time copy of the database to dest +// using SQLite's `VACUUM INTO`. +// +// This is not a convenience wrapper around a file copy — it is the only correct +// way to snapshot this database. capstan.db runs in WAL mode (see the +// journal_mode pragma above), so the .db file on disk is not self-contained: +// recent commits live in the -wal sidecar until a checkpoint. Copying the file +// while writes are in flight yields a torn or stale database. VACUUM INTO runs +// through the SQL layer, so it observes a single consistent snapshot and emits +// a standalone, already-compacted database with no sidecar files. +// +// It does not block writers. +// +// dest must not already exist — SQLite refuses to overwrite, and that refusal +// is deliberate rather than something to work around: silently clobbering a +// previous snapshot would destroy the artifact a restore depends on. Callers +// remove a stale file explicitly if they intend to replace it. +func (d *DB) VacuumInto(dest string) error { + // Parameter binding is not permitted for VACUUM INTO in SQLite, so the path + // is interpolated. Callers construct dest from configuration, never from + // user input; the quote-doubling keeps a path containing a single quote from + // terminating the literal. + quoted := "'" + strings.ReplaceAll(dest, "'", "''") + "'" + if _, err := d.db.Exec("VACUUM INTO " + quoted); err != nil { + return fmt.Errorf("vacuum into %s: %w", dest, err) + } + return nil +} diff --git a/backend/internal/database/database_vacuum_test.go b/backend/internal/database/database_vacuum_test.go new file mode 100644 index 0000000..04c0a3a --- /dev/null +++ b/backend/internal/database/database_vacuum_test.go @@ -0,0 +1,72 @@ +package database + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestVacuumInto_ProducesAnOpenableStandaloneCopy pins the primitive the backup +// engine relies on (agent-os-36o). +// +// The point is not that a file appears — it is that the file is a complete, +// self-contained database containing data committed right up to the snapshot. +// capstan.db runs in WAL mode, so a plain file copy can miss recent commits +// that are still in the -wal sidecar. This asserts the copy round-trips through +// a fresh connection and carries the data. +func TestVacuumInto_ProducesAnOpenableStandaloneCopy(t *testing.T) { + t.Parallel() + + src, err := NewWithMigrations(":memory:") + require.NoError(t, err) + t.Cleanup(func() { src.Close() }) + + require.NoError(t, src.SetSetting("canary", "vacuum-into-round-trip")) + + // Stage the copy as /capstan.db so it can be reopened by passing + // to NewWithMigrations, which is exactly how a restore puts it back. + restoreDir := filepath.Join(t.TempDir(), "restored") + require.NoError(t, os.MkdirAll(restoreDir, 0o700)) + dest := filepath.Join(restoreDir, "capstan.db") + require.NoError(t, src.VacuumInto(dest)) + + info, statErr := os.Stat(dest) + require.NoError(t, statErr, "the snapshot file must exist") + assert.Greater(t, info.Size(), int64(0), "the snapshot must not be empty") + + // No sidecars: VACUUM INTO emits a standalone database, which is what makes + // it safe to hand to restic as a single file. + for _, sidecar := range []string{dest + "-wal", dest + "-shm"} { + _, sErr := os.Stat(sidecar) + assert.True(t, os.IsNotExist(sErr), "%s must not exist alongside the snapshot", sidecar) + } + + // Reopen the copy independently and confirm the committed row is present. + restored, err := NewWithMigrations(restoreDir) + require.NoError(t, err, "the snapshot must be openable as a database in its own right") + t.Cleanup(func() { restored.Close() }) + + got, err := restored.GetSetting("canary") + require.NoError(t, err) + assert.Equal(t, "vacuum-into-round-trip", got, + "data committed before the snapshot must survive into the copy") +} + +// TestVacuumInto_RefusesToOverwrite documents that SQLite will not clobber an +// existing destination. Callers must remove a stale file deliberately — +// silently overwriting would destroy the artifact a restore depends on. +func TestVacuumInto_RefusesToOverwrite(t *testing.T) { + t.Parallel() + + src, err := NewWithMigrations(":memory:") + require.NoError(t, err) + t.Cleanup(func() { src.Close() }) + + dest := filepath.Join(t.TempDir(), "snapshot.db") + require.NoError(t, os.WriteFile(dest, []byte("pre-existing"), 0o600)) + + assert.Error(t, src.VacuumInto(dest), "VACUUM INTO must refuse an existing destination") +} diff --git a/backend/internal/services/backup.go b/backend/internal/services/backup.go index baa6c72..6f71063 100644 --- a/backend/internal/services/backup.go +++ b/backend/internal/services/backup.go @@ -40,6 +40,22 @@ const ( TriggerScheduled = "scheduled" // automatic, from the backup scheduler ) +// DatabaseBackupTag is the restic tag carried by the capstan.db snapshot. +// +// It is deliberately not a stack ID. Stack snapshots are tagged with the stack +// they came from, so tagging the database distinctly keeps it selectable in the +// Snapshots UI and impossible to mistake for a stack — and lets a restore +// target it directly with `restic snapshots --tag capstan-database`. +const DatabaseBackupTag = "capstan-database" + +// databaseStagingDir is the directory under DataDir where the database snapshot +// is staged before restic picks it up. +// +// The path is fixed rather than a random temp dir on purpose: it becomes the +// path recorded inside the restic snapshot, and therefore the path an operator +// types during a restore. A random path would make the runbook unwritable. +const databaseStagingDir = "backup-staging" + // BackupAvailability describes which parts of the engine are functional. type BackupAvailability struct { ResticPresent bool `json:"resticPresent"` @@ -420,6 +436,19 @@ func (s *BackupService) RunBackup( run.StacksTotal = len(policies) var totalBytesAdded int64 + + // The database goes first, deliberately. It is the artifact without which a + // restore produces an empty Capstan, and capturing it before the per-stack + // loop means a stack failure part-way through still leaves it in the + // repository. See backupDatabase for the full rationale (agent-os-36o). + dbFailed := false + if dbBytes, dbErr := s.backupDatabase(ctx, restic, dryRun, out); dbErr != nil { + dbFailed = true + stream(out, "error", fmt.Sprintf("database snapshot failed: %v", dbErr)) + } else { + totalBytesAdded += dbBytes + } + for _, policy := range policies { stackID := policy.TargetID itemBytes, itemErr := s.backupStack(ctx, restic, stackID, policy.StopPolicy, dryRun, run.ID, out) @@ -455,6 +484,17 @@ func (s *BackupService) RunBackup( run.Status = "partial" } + // A run that saved every stack but lost the database is not a success. It + // would look green in the UI while leaving the single most important + // artifact out of the repository, which is exactly the silent failure + // agent-os-36o exists to remove. + if dbFailed && run.Status == "success" { + run.Status = "partial" + if run.ErrorMessage == "" { + run.ErrorMessage = "database snapshot failed; stack backups succeeded" + } + } + s.finaliseRun(run) stream(out, "info", fmt.Sprintf("Backup run finished: status=%s ok=%d failed=%d", run.Status, run.StacksOK, run.StacksFailed)) @@ -527,6 +567,19 @@ func (s *BackupService) RunBackupWithRunID( run.StacksTotal = len(policies) var totalBytesAdded int64 + + // The database goes first, deliberately. It is the artifact without which a + // restore produces an empty Capstan, and capturing it before the per-stack + // loop means a stack failure part-way through still leaves it in the + // repository. See backupDatabase for the full rationale (agent-os-36o). + dbFailed := false + if dbBytes, dbErr := s.backupDatabase(ctx, restic, dryRun, out); dbErr != nil { + dbFailed = true + stream(out, "error", fmt.Sprintf("database snapshot failed: %v", dbErr)) + } else { + totalBytesAdded += dbBytes + } + for _, policy := range policies { stackID := policy.TargetID itemBytes, itemErr := s.backupStack(ctx, restic, stackID, policy.StopPolicy, dryRun, run.ID, out) @@ -554,6 +607,17 @@ func (s *BackupService) RunBackupWithRunID( run.Status = "partial" } + // A run that saved every stack but lost the database is not a success. It + // would look green in the UI while leaving the single most important + // artifact out of the repository, which is exactly the silent failure + // agent-os-36o exists to remove. + if dbFailed && run.Status == "success" { + run.Status = "partial" + if run.ErrorMessage == "" { + run.ErrorMessage = "database snapshot failed; stack backups succeeded" + } + } + s.finaliseRun(run) stream(out, "info", fmt.Sprintf("Backup run finished: status=%s ok=%d failed=%d", run.Status, run.StacksOK, run.StacksFailed)) @@ -1058,3 +1122,89 @@ func drainOut(wg *sync.WaitGroup, src <-chan StreamLine, dst chan<- StreamLine) stream(dst, line.Type, line.Line) } } + +// DatabaseSnapshotPath returns the absolute path the capstan.db snapshot is +// staged at, and therefore the path recorded inside the restic snapshot. +// +// Exported because the disaster-recovery runbook has to name it exactly: a +// restore writes to /, and an operator following the runbook +// under pressure should not have to derive it. +func (s *BackupService) DatabaseSnapshotPath() string { + return filepath.Join(s.cfg.DataDir, databaseStagingDir, "capstan.db") +} + +// backupDatabase snapshots capstan.db and hands the artifact to restic under +// DatabaseBackupTag. +// +// Why this exists (agent-os-36o): the backup engine's scope was each stack's +// compose directory. capstan.db — user accounts, the encrypted git tokens and +// restic password, every setting, the backup and auto-update policies, the +// stacks registry and the audit log — was in no snapshot at all. Worse, the +// restic repository defaults to /restic-repo, the same volume the +// database lives on, so losing that one volume lost the database and the local +// snapshots together. The only surviving copy was whatever rclone had synced +// offsite, which contained stack directories and nothing else. +// +// The snapshot is taken with VACUUM INTO rather than by copying the file: +// capstan.db runs in WAL mode, so a file copy taken while writes are in flight +// can be torn or missing recent commits. See database.DB.VacuumInto. +// +// NOTE ON SECRETS: the secrets inside are encrypted with a key derived from +// STORAGE_KEY, which lives in the environment and is deliberately NOT in the +// backup. A stolen backup therefore does not yield the git tokens — but it also +// means restoring onto a host with a different STORAGE_KEY produces a database +// whose secrets cannot be decrypted. The runbook states this; see README. +func (s *BackupService) backupDatabase( + ctx context.Context, + restic *ResticManager, + dryRun bool, + out chan<- StreamLine, +) (bytesAdded int64, retErr error) { + dest := s.DatabaseSnapshotPath() + + if dryRun { + stream(out, "info", fmt.Sprintf("[database] dry run: would snapshot capstan.db to %s", dest)) + return 0, nil + } + + stream(out, "info", "[database] snapshotting capstan.db (VACUUM INTO)") + + if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil { + return 0, fmt.Errorf("create staging dir: %w", err) + } + + // VACUUM INTO refuses to overwrite. A file here is a leftover from a run + // that died before its cleanup, so removing it is correct — but only ever + // this exact path, never a directory. + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + return 0, fmt.Errorf("remove stale staged snapshot: %w", err) + } + + if err := s.db.VacuumInto(dest); err != nil { + return 0, fmt.Errorf("snapshot database: %w", err) + } + + // The staged copy is a full plaintext database. It must not outlive the run, + // whatever happens next. + defer func() { + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + stream(out, "error", fmt.Sprintf("[database] failed to remove staged snapshot %s: %v", dest, err)) + if retErr == nil { + retErr = fmt.Errorf("remove staged snapshot: %w", err) + } + } + }() + + summary, err := restic.Backup(ctx, dest, []string{DatabaseBackupTag}, out) + if err != nil { + return 0, fmt.Errorf("restic backup database: %w", err) + } + + if summary != nil { + bytesAdded = summary.BytesAdded + stream(out, "info", fmt.Sprintf("[database] snapshot %s captured (%d bytes added)", + summary.SnapshotID, summary.BytesAdded)) + } + + return bytesAdded, nil +} diff --git a/backend/internal/services/backup_test.go b/backend/internal/services/backup_test.go index 7979cbe..13c37e7 100644 --- a/backend/internal/services/backup_test.go +++ b/backend/internal/services/backup_test.go @@ -6,6 +6,7 @@ import ( "errors" "log/slog" "net/http" + "os" "path/filepath" "sync" "testing" @@ -542,8 +543,16 @@ func TestRunBackup_PerStackFailureIsolation_Partial(t *testing.T) { runner := &fakeRunner{} // unused; multiCallRunner below drives all calls // Use a multi-call runner: fail stack-a, succeed stack-b. + // + // The sequence is positional, so it must account for the capstan.db snapshot + // that every run now takes BEFORE the per-stack loop (agent-os-36o). Without + // this leading entry the database call would consume stack-a's injected + // failure and the isolation this test checks would silently stop being + // exercised. multiRunner := &multiCallRunner{ responses: []multiCallResponse{ + // database snapshot → success + {binary: "restic", argPrefix: "backup", err: nil}, // stack-a backup call → error {binary: "restic", argPrefix: "backup", err: errors.New("disk full")}, // stack-b backup call → success @@ -1864,3 +1873,167 @@ func TestRunRestore_HappyPath_EscapingSubdirIsRejected(t *testing.T) { } } } + +// ============================================================ +// Database snapshot in every backup run (agent-os-36o) +// ============================================================ + +// findCall returns the first recorded call whose args start with subcommand and +// contain every string in mustContain. +func findCall(calls []fakeCall, subcommand string, mustContain ...string) (fakeCall, bool) { + for _, c := range calls { + if len(c.Args) == 0 || c.Args[0] != subcommand { + continue + } + ok := true + for _, w := range mustContain { + found := false + for _, a := range c.Args { + if a == w { + found = true + break + } + } + if !found { + ok = false + break + } + } + if ok { + return c, true + } + } + return fakeCall{}, false +} + +// TestRunBackup_CapturesDatabaseUnderItsOwnTag pins the core of agent-os-36o: +// capstan.db is captured on every run, under a tag that is not a stack ID. +// +// Before this, the backup engine's scope was each stack's compose directory and +// the database — accounts, encrypted git tokens and restic password, settings, +// policies, history — was in no snapshot at all. +func TestRunBackup_CapturesDatabaseUnderItsOwnTag(t *testing.T) { + t.Parallel() + + db := newBackupTestDB(t) + runner := &fakeRunner{outputData: snapshotJSON("abc123", "abc", "myapp")} + svc := buildSvc(t, db, &fakeDocker{}, runner, runner) + seedStack(t, db, "myapp", "hot") + + out := make(chan StreamLine, 256) + run, err := svc.RunBackup(context.Background(), nil, false, TriggerManual, out) + require.NoError(t, err) + assert.Equal(t, "success", run.Status) + + call, ok := findCall(runner.calls, "backup", "--tag", DatabaseBackupTag) + require.True(t, ok, "every backup run must capture capstan.db under the %q tag", DatabaseBackupTag) + + assert.Contains(t, call.Args, svc.DatabaseSnapshotPath(), + "the database snapshot must be taken from the staged VACUUM INTO copy") + + // The tag must not collide with a stack ID, or the snapshot becomes + // selectable as if it were a stack. + assert.NotContains(t, call.Args, "myapp", + "the database snapshot must not carry a stack tag") +} + +// TestRunBackup_StagedDatabaseCopyIsRemoved verifies the staged artifact does +// not outlive the run. It is a full plaintext database sitting on disk. +func TestRunBackup_StagedDatabaseCopyIsRemoved(t *testing.T) { + t.Parallel() + + db := newBackupTestDB(t) + runner := &fakeRunner{outputData: snapshotJSON("abc123", "abc", "myapp")} + svc := buildSvc(t, db, &fakeDocker{}, runner, runner) + seedStack(t, db, "myapp", "hot") + + out := make(chan StreamLine, 256) + _, err := svc.RunBackup(context.Background(), nil, false, TriggerManual, out) + require.NoError(t, err) + + _, statErr := os.Stat(svc.DatabaseSnapshotPath()) + assert.True(t, os.IsNotExist(statErr), + "the staged plaintext database copy must be removed after the run, got err=%v", statErr) +} + +// TestRunBackup_DryRunDoesNotStageDatabase verifies a dry run neither writes the +// staged copy nor hands anything to restic. +func TestRunBackup_DryRunDoesNotStageDatabase(t *testing.T) { + t.Parallel() + + db := newBackupTestDB(t) + runner := &fakeRunner{outputData: snapshotJSON("abc123", "abc", "myapp")} + svc := buildSvc(t, db, &fakeDocker{}, runner, runner) + seedStack(t, db, "myapp", "hot") + + out := make(chan StreamLine, 256) + _, err := svc.RunBackup(context.Background(), nil, true, TriggerManual, out) + require.NoError(t, err) + + _, ok := findCall(runner.calls, "backup", "--tag", DatabaseBackupTag) + assert.False(t, ok, "a dry run must not hand the database to restic") + + _, statErr := os.Stat(svc.DatabaseSnapshotPath()) + assert.True(t, os.IsNotExist(statErr), "a dry run must not stage a database copy") +} + +// databaseFailingRunner fails only the restic invocation that carries the +// database tag, so a test can isolate a database-snapshot failure from a +// stack-backup failure. +type databaseFailingRunner struct { + fakeRunner +} + +func (f *databaseFailingRunner) Run( + ctx context.Context, + name string, + args []string, + env []string, + out chan<- StreamLine, +) error { + for _, a := range args { + if a == DatabaseBackupTag { + f.fakeRunner.calls = append(f.fakeRunner.calls, fakeCall{Binary: name, Args: args, Env: env}) + return errors.New("injected database snapshot failure") + } + } + return f.fakeRunner.Run(ctx, name, args, env, out) +} + +// TestRunBackup_DatabaseFailureDowngradesSuccessToPartial pins the rule that a +// run which saved every stack but lost the database is NOT reported as success. +// +// Without this, the most important artifact could go missing while the UI stayed +// green — the silent failure agent-os-36o exists to remove. +func TestRunBackup_DatabaseFailureDowngradesSuccessToPartial(t *testing.T) { + t.Parallel() + + db := newBackupTestDB(t) + runner := &databaseFailingRunner{fakeRunner: fakeRunner{outputData: snapshotJSON("abc123", "abc", "myapp")}} + svc := buildSvc(t, db, &fakeDocker{}, runner, runner) + seedStack(t, db, "myapp", "hot") + + out := make(chan StreamLine, 256) + run, err := svc.RunBackup(context.Background(), nil, false, TriggerManual, out) + require.NoError(t, err) + + assert.Equal(t, 1, run.StacksOK, "the stack backup itself should have succeeded") + assert.Equal(t, 0, run.StacksFailed) + assert.Equal(t, "partial", run.Status, + "a run that lost the database must not report success") + assert.NotEmpty(t, run.ErrorMessage, "the run must say why it was downgraded") +} + +// TestDatabaseSnapshotPath_IsUnderDataDir pins the path the runbook names. An +// operator restoring under pressure reads this path out of README; if it moves +// silently, the runbook is wrong at the worst possible moment. +func TestDatabaseSnapshotPath_IsUnderDataDir(t *testing.T) { + t.Parallel() + + db := newBackupTestDB(t) + svc := buildSvc(t, db, &fakeDocker{}, &fakeRunner{}, &fakeRunner{}) + + assert.Equal(t, + filepath.Join(svc.Config().DataDir, "backup-staging", "capstan.db"), + svc.DatabaseSnapshotPath()) +}