diff --git a/README.md b/README.md index c4f9746..439d605 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,35 @@ Capstan includes a built-in backup engine powered by [restic](https://restic.net ### What gets backed up -Each stack's compose directory is backed up as a restic snapshot tagged with the stack ID. +Every backup run captures two kinds of thing: + +| Artifact | restic tag | Notes | +| --- | --- | --- | +| Each stack's compose directory | the stack ID | One snapshot per enabled stack | +| `capstan.db` | `capstan-database` | Captured on **every** run, before the stacks | + +The database is what makes an instance *this* instance: user accounts, the +encrypted git tokens and restic password, every setting, backup and auto-update +policies, the stacks registry, and the audit log. Without it a restore gives you +your compose files back and nothing else. + +It is snapshotted with SQLite's `VACUUM INTO` rather than copied. `capstan.db` +runs in WAL mode, so a plain file copy taken while writes are in flight can be +torn or missing recent commits. + +A run that saves every stack but fails to capture the database is reported as +**partial**, never success. + Backups can be triggered manually (Settings, Backup tab) or run on a schedule. +> **`STORAGE_KEY` is not in the backup, and that is deliberate.** The secrets +> inside `capstan.db` are encrypted with a key derived from it, so a stolen +> backup does not yield your git tokens. The corollary is that restoring onto a +> host with a *different* `STORAGE_KEY` gives you a database whose secrets +> cannot be decrypted. Store `STORAGE_KEY` and `JWT_SECRET` in the same password +> manager as the restic repository password. All three are needed for a complete +> recovery. + ### Configuration The quickest path is the **Settings UI** (Settings, Backup). All fields save to the @@ -157,15 +183,146 @@ curl -X POST http://localhost:5001/api/v1/backups/restore \ ### Disaster recovery -If the host is lost, recover from an rclone remote: +Follow this in order. **The database is restored before the stacks** — restoring +stacks first gives you a Capstan that does not know they exist. + +#### Before you start, you need + +| Item | Why | +| --- | --- | +| The restic repository password | Without it the repository cannot be opened at all | +| `STORAGE_KEY` | The old value, or every stored secret is unreadable | +| `JWT_SECRET` | A different value invalidates existing sessions (survivable) | +| Access to the rclone remote | The only copy that survives losing the data volume | + +If you have the repository but not `STORAGE_KEY`, you can still recover +everything except the encrypted secrets — see step 6. + +#### 1. Deploy a fresh instance, stopped + +Bring up a new Capstan with the same `./data` bind mount path and the **same** +`STORAGE_KEY` and `JWT_SECRET` as the lost instance, then stop the container: + +```bash +docker compose -f docker-compose.prod.yaml up -d +docker compose -f docker-compose.prod.yaml stop capstan +``` + +Stopping matters. Overwriting `capstan.db` underneath a running server corrupts +it — the server holds the WAL open and will write over what you just restored. + +#### Run steps 2–4 inside the image, not on the host + +Every command below is run in a throwaway container built from the Capstan +image, with the same volumes. Three reasons, all learned the hard way in a real +drill: + +- `restic` and `rclone` ship **inside** the image at pinned versions. A freshly + provisioned recovery host usually has neither, and installing them mid-outage + is a poor use of the outage. +- The rclone remote path stored in Capstan's settings is a path **as seen from + inside the container**. Running the same `rclone sync` on the host resolves it + against the host filesystem and fails with `directory not found`. +- The paths inside the restic snapshot are container paths, so restoring in the + same namespace keeps them meaningful. + +Define this once, adjusting the paths to your deployment: + +```bash +alias capstan-tools='docker run --rm \ + -v ./data:/app/data \ + -v ~/.config/rclone/rclone.conf:/home/appuser/.config/rclone/rclone.conf:ro \ + --entrypoint sh ghcr.io/thinkbig1979/capstan:latest -c' +``` + +#### 2. Get the repository back from the remote + +```bash +capstan-tools 'rclone sync : /app/data/restic-repo --stats-one-line' +``` + +`:` are exactly the values shown in Settings → Backup. This is +what the UI's DR Restore does; doing it by hand keeps step 1's "container +stopped" rule intact. + +#### 3. Find the database snapshot + +```bash +capstan-tools ' + printf "%s" "$RESTIC_PASSWORD" > /tmp/pw + export RESTIC_REPOSITORY=/app/data/restic-repo RESTIC_PASSWORD_FILE=/tmp/pw + restic snapshots --tag capstan-database' +``` + +The newest one is normally what you want. Note its short ID. + +#### 4. Restore the database + +The snapshot contains the file at `/app/data/backup-staging/capstan.db`, so a +restore reproduces that path underneath `--target`: + +```bash +capstan-tools ' + set -e + printf "%s" "$RESTIC_PASSWORD" > /tmp/pw + export RESTIC_REPOSITORY=/app/data/restic-repo RESTIC_PASSWORD_FILE=/tmp/pw + restic restore --target /tmp/capstan-restore + cp /tmp/capstan-restore/app/data/backup-staging/capstan.db /app/data/capstan.db + rm -rf /tmp/capstan-restore + rm -f /app/data/capstan.db-wal /app/data/capstan.db-shm + chown appuser:appuser /app/data/capstan.db' +``` + +Two details that will bite if skipped: + +- **Delete `capstan.db-wal` and `capstan.db-shm`.** They belong to the database + you just replaced. Left in place, SQLite tries to apply them to the restored + file. +- **`chown appuser:appuser`.** The restore runs as root; Capstan runs as + `appuser` (UID 1000 by default, remappable via `PUID`/`PGID`). A root-owned + `capstan.db` leaves the server unable to write to its own database. + +#### 5. Start Capstan and verify before going further + +```bash +docker compose -f docker-compose.prod.yaml start capstan +``` + +Check all four, in this order. Each one fails differently: + +1. **Log in** with a pre-existing account. Failure here means the wrong + `capstan.db`, not a `STORAGE_KEY` problem. +2. **Settings → Backup** shows your repository path, retention and schedule. + Empty defaults mean the database did not actually land. +3. **A git-backed stack can pull.** This is the `STORAGE_KEY` test — the token + is decrypted to run it. An authentication failure here with the right + password means the key differs from the one that encrypted it. +4. **Settings → Backup → History** lists runs from before the incident, and your + backup policies are still enabled. + +#### 6. If `STORAGE_KEY` was lost + +Everything except the encrypted secrets is recoverable. Accounts, settings, +policies, history and the stacks registry all come back. What must be re-entered +by hand: the restic repository password, git HTTPS tokens, and any other stored +credential. Re-enter them in Settings; the new values are encrypted with the new +key. + +#### 7. Restore the stacks + +Only now, with a working Capstan that knows about them, restore the compose +directories from Settings → Backup → Snapshots, or via the API: + +```bash +curl -X POST http://localhost:5001/api/v1/backups/restore \ + -H 'Content-Type: application/json' \ + -d '{"stackId":"","snapshotId":"","confirm":true}' +``` -1. Deploy a fresh Capstan instance with the same `./data` bind mount path. -2. Trigger a DR restore (Settings, Backup, DR Restore). This syncs the full - restic repository from the configured remote back to `/app/data/restic-repo`. -3. Restore individual stacks via the Snapshots panel. +#### Practise this -The restic repository password is required for DR recovery. Store it separately -from the server (e.g. a password manager). +A runbook that has never been executed is a hypothesis. Run it end to end +against a scratch instance at least once, before you need it. ## Volume Path Identity diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go index ae34753..4aeff60 100644 --- a/backend/internal/database/database.go +++ b/backend/internal/database/database.go @@ -2,7 +2,9 @@ package database import ( "database/sql" + "fmt" "os" + "strings" "time" _ "modernc.org/sqlite" @@ -92,3 +94,32 @@ func NewWithMigrationsAndEncryptor(dataDir string, encryptor TokenEncryptor) (*D func (d *DB) Close() error { return d.db.Close() } + +// VacuumInto writes a consistent point-in-time copy of the database to dest +// using SQLite's `VACUUM INTO`. +// +// This is not a convenience wrapper around a file copy — it is the only correct +// way to snapshot this database. capstan.db runs in WAL mode (see the +// journal_mode pragma above), so the .db file on disk is not self-contained: +// recent commits live in the -wal sidecar until a checkpoint. Copying the file +// while writes are in flight yields a torn or stale database. VACUUM INTO runs +// through the SQL layer, so it observes a single consistent snapshot and emits +// a standalone, already-compacted database with no sidecar files. +// +// It does not block writers. +// +// dest must not already exist — SQLite refuses to overwrite, and that refusal +// is deliberate rather than something to work around: silently clobbering a +// previous snapshot would destroy the artifact a restore depends on. Callers +// remove a stale file explicitly if they intend to replace it. +func (d *DB) VacuumInto(dest string) error { + // Parameter binding is not permitted for VACUUM INTO in SQLite, so the path + // is interpolated. Callers construct dest from configuration, never from + // user input; the quote-doubling keeps a path containing a single quote from + // terminating the literal. + quoted := "'" + strings.ReplaceAll(dest, "'", "''") + "'" + if _, err := d.db.Exec("VACUUM INTO " + quoted); err != nil { + return fmt.Errorf("vacuum into %s: %w", dest, err) + } + return nil +} diff --git a/backend/internal/database/database_vacuum_test.go b/backend/internal/database/database_vacuum_test.go new file mode 100644 index 0000000..04c0a3a --- /dev/null +++ b/backend/internal/database/database_vacuum_test.go @@ -0,0 +1,72 @@ +package database + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestVacuumInto_ProducesAnOpenableStandaloneCopy pins the primitive the backup +// engine relies on (agent-os-36o). +// +// The point is not that a file appears — it is that the file is a complete, +// self-contained database containing data committed right up to the snapshot. +// capstan.db runs in WAL mode, so a plain file copy can miss recent commits +// that are still in the -wal sidecar. This asserts the copy round-trips through +// a fresh connection and carries the data. +func TestVacuumInto_ProducesAnOpenableStandaloneCopy(t *testing.T) { + t.Parallel() + + src, err := NewWithMigrations(":memory:") + require.NoError(t, err) + t.Cleanup(func() { src.Close() }) + + require.NoError(t, src.SetSetting("canary", "vacuum-into-round-trip")) + + // Stage the copy as /capstan.db so it can be reopened by passing + // to NewWithMigrations, which is exactly how a restore puts it back. + restoreDir := filepath.Join(t.TempDir(), "restored") + require.NoError(t, os.MkdirAll(restoreDir, 0o700)) + dest := filepath.Join(restoreDir, "capstan.db") + require.NoError(t, src.VacuumInto(dest)) + + info, statErr := os.Stat(dest) + require.NoError(t, statErr, "the snapshot file must exist") + assert.Greater(t, info.Size(), int64(0), "the snapshot must not be empty") + + // No sidecars: VACUUM INTO emits a standalone database, which is what makes + // it safe to hand to restic as a single file. + for _, sidecar := range []string{dest + "-wal", dest + "-shm"} { + _, sErr := os.Stat(sidecar) + assert.True(t, os.IsNotExist(sErr), "%s must not exist alongside the snapshot", sidecar) + } + + // Reopen the copy independently and confirm the committed row is present. + restored, err := NewWithMigrations(restoreDir) + require.NoError(t, err, "the snapshot must be openable as a database in its own right") + t.Cleanup(func() { restored.Close() }) + + got, err := restored.GetSetting("canary") + require.NoError(t, err) + assert.Equal(t, "vacuum-into-round-trip", got, + "data committed before the snapshot must survive into the copy") +} + +// TestVacuumInto_RefusesToOverwrite documents that SQLite will not clobber an +// existing destination. Callers must remove a stale file deliberately — +// silently overwriting would destroy the artifact a restore depends on. +func TestVacuumInto_RefusesToOverwrite(t *testing.T) { + t.Parallel() + + src, err := NewWithMigrations(":memory:") + require.NoError(t, err) + t.Cleanup(func() { src.Close() }) + + dest := filepath.Join(t.TempDir(), "snapshot.db") + require.NoError(t, os.WriteFile(dest, []byte("pre-existing"), 0o600)) + + assert.Error(t, src.VacuumInto(dest), "VACUUM INTO must refuse an existing destination") +} diff --git a/backend/internal/services/backup.go b/backend/internal/services/backup.go index baa6c72..6f71063 100644 --- a/backend/internal/services/backup.go +++ b/backend/internal/services/backup.go @@ -40,6 +40,22 @@ const ( TriggerScheduled = "scheduled" // automatic, from the backup scheduler ) +// DatabaseBackupTag is the restic tag carried by the capstan.db snapshot. +// +// It is deliberately not a stack ID. Stack snapshots are tagged with the stack +// they came from, so tagging the database distinctly keeps it selectable in the +// Snapshots UI and impossible to mistake for a stack — and lets a restore +// target it directly with `restic snapshots --tag capstan-database`. +const DatabaseBackupTag = "capstan-database" + +// databaseStagingDir is the directory under DataDir where the database snapshot +// is staged before restic picks it up. +// +// The path is fixed rather than a random temp dir on purpose: it becomes the +// path recorded inside the restic snapshot, and therefore the path an operator +// types during a restore. A random path would make the runbook unwritable. +const databaseStagingDir = "backup-staging" + // BackupAvailability describes which parts of the engine are functional. type BackupAvailability struct { ResticPresent bool `json:"resticPresent"` @@ -420,6 +436,19 @@ func (s *BackupService) RunBackup( run.StacksTotal = len(policies) var totalBytesAdded int64 + + // The database goes first, deliberately. It is the artifact without which a + // restore produces an empty Capstan, and capturing it before the per-stack + // loop means a stack failure part-way through still leaves it in the + // repository. See backupDatabase for the full rationale (agent-os-36o). + dbFailed := false + if dbBytes, dbErr := s.backupDatabase(ctx, restic, dryRun, out); dbErr != nil { + dbFailed = true + stream(out, "error", fmt.Sprintf("database snapshot failed: %v", dbErr)) + } else { + totalBytesAdded += dbBytes + } + for _, policy := range policies { stackID := policy.TargetID itemBytes, itemErr := s.backupStack(ctx, restic, stackID, policy.StopPolicy, dryRun, run.ID, out) @@ -455,6 +484,17 @@ func (s *BackupService) RunBackup( run.Status = "partial" } + // A run that saved every stack but lost the database is not a success. It + // would look green in the UI while leaving the single most important + // artifact out of the repository, which is exactly the silent failure + // agent-os-36o exists to remove. + if dbFailed && run.Status == "success" { + run.Status = "partial" + if run.ErrorMessage == "" { + run.ErrorMessage = "database snapshot failed; stack backups succeeded" + } + } + s.finaliseRun(run) stream(out, "info", fmt.Sprintf("Backup run finished: status=%s ok=%d failed=%d", run.Status, run.StacksOK, run.StacksFailed)) @@ -527,6 +567,19 @@ func (s *BackupService) RunBackupWithRunID( run.StacksTotal = len(policies) var totalBytesAdded int64 + + // The database goes first, deliberately. It is the artifact without which a + // restore produces an empty Capstan, and capturing it before the per-stack + // loop means a stack failure part-way through still leaves it in the + // repository. See backupDatabase for the full rationale (agent-os-36o). + dbFailed := false + if dbBytes, dbErr := s.backupDatabase(ctx, restic, dryRun, out); dbErr != nil { + dbFailed = true + stream(out, "error", fmt.Sprintf("database snapshot failed: %v", dbErr)) + } else { + totalBytesAdded += dbBytes + } + for _, policy := range policies { stackID := policy.TargetID itemBytes, itemErr := s.backupStack(ctx, restic, stackID, policy.StopPolicy, dryRun, run.ID, out) @@ -554,6 +607,17 @@ func (s *BackupService) RunBackupWithRunID( run.Status = "partial" } + // A run that saved every stack but lost the database is not a success. It + // would look green in the UI while leaving the single most important + // artifact out of the repository, which is exactly the silent failure + // agent-os-36o exists to remove. + if dbFailed && run.Status == "success" { + run.Status = "partial" + if run.ErrorMessage == "" { + run.ErrorMessage = "database snapshot failed; stack backups succeeded" + } + } + s.finaliseRun(run) stream(out, "info", fmt.Sprintf("Backup run finished: status=%s ok=%d failed=%d", run.Status, run.StacksOK, run.StacksFailed)) @@ -1058,3 +1122,89 @@ func drainOut(wg *sync.WaitGroup, src <-chan StreamLine, dst chan<- StreamLine) stream(dst, line.Type, line.Line) } } + +// DatabaseSnapshotPath returns the absolute path the capstan.db snapshot is +// staged at, and therefore the path recorded inside the restic snapshot. +// +// Exported because the disaster-recovery runbook has to name it exactly: a +// restore writes to /, and an operator following the runbook +// under pressure should not have to derive it. +func (s *BackupService) DatabaseSnapshotPath() string { + return filepath.Join(s.cfg.DataDir, databaseStagingDir, "capstan.db") +} + +// backupDatabase snapshots capstan.db and hands the artifact to restic under +// DatabaseBackupTag. +// +// Why this exists (agent-os-36o): the backup engine's scope was each stack's +// compose directory. capstan.db — user accounts, the encrypted git tokens and +// restic password, every setting, the backup and auto-update policies, the +// stacks registry and the audit log — was in no snapshot at all. Worse, the +// restic repository defaults to /restic-repo, the same volume the +// database lives on, so losing that one volume lost the database and the local +// snapshots together. The only surviving copy was whatever rclone had synced +// offsite, which contained stack directories and nothing else. +// +// The snapshot is taken with VACUUM INTO rather than by copying the file: +// capstan.db runs in WAL mode, so a file copy taken while writes are in flight +// can be torn or missing recent commits. See database.DB.VacuumInto. +// +// NOTE ON SECRETS: the secrets inside are encrypted with a key derived from +// STORAGE_KEY, which lives in the environment and is deliberately NOT in the +// backup. A stolen backup therefore does not yield the git tokens — but it also +// means restoring onto a host with a different STORAGE_KEY produces a database +// whose secrets cannot be decrypted. The runbook states this; see README. +func (s *BackupService) backupDatabase( + ctx context.Context, + restic *ResticManager, + dryRun bool, + out chan<- StreamLine, +) (bytesAdded int64, retErr error) { + dest := s.DatabaseSnapshotPath() + + if dryRun { + stream(out, "info", fmt.Sprintf("[database] dry run: would snapshot capstan.db to %s", dest)) + return 0, nil + } + + stream(out, "info", "[database] snapshotting capstan.db (VACUUM INTO)") + + if err := os.MkdirAll(filepath.Dir(dest), 0o700); err != nil { + return 0, fmt.Errorf("create staging dir: %w", err) + } + + // VACUUM INTO refuses to overwrite. A file here is a leftover from a run + // that died before its cleanup, so removing it is correct — but only ever + // this exact path, never a directory. + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + return 0, fmt.Errorf("remove stale staged snapshot: %w", err) + } + + if err := s.db.VacuumInto(dest); err != nil { + return 0, fmt.Errorf("snapshot database: %w", err) + } + + // The staged copy is a full plaintext database. It must not outlive the run, + // whatever happens next. + defer func() { + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + stream(out, "error", fmt.Sprintf("[database] failed to remove staged snapshot %s: %v", dest, err)) + if retErr == nil { + retErr = fmt.Errorf("remove staged snapshot: %w", err) + } + } + }() + + summary, err := restic.Backup(ctx, dest, []string{DatabaseBackupTag}, out) + if err != nil { + return 0, fmt.Errorf("restic backup database: %w", err) + } + + if summary != nil { + bytesAdded = summary.BytesAdded + stream(out, "info", fmt.Sprintf("[database] snapshot %s captured (%d bytes added)", + summary.SnapshotID, summary.BytesAdded)) + } + + return bytesAdded, nil +} diff --git a/backend/internal/services/backup_test.go b/backend/internal/services/backup_test.go index 7979cbe..13c37e7 100644 --- a/backend/internal/services/backup_test.go +++ b/backend/internal/services/backup_test.go @@ -6,6 +6,7 @@ import ( "errors" "log/slog" "net/http" + "os" "path/filepath" "sync" "testing" @@ -542,8 +543,16 @@ func TestRunBackup_PerStackFailureIsolation_Partial(t *testing.T) { runner := &fakeRunner{} // unused; multiCallRunner below drives all calls // Use a multi-call runner: fail stack-a, succeed stack-b. + // + // The sequence is positional, so it must account for the capstan.db snapshot + // that every run now takes BEFORE the per-stack loop (agent-os-36o). Without + // this leading entry the database call would consume stack-a's injected + // failure and the isolation this test checks would silently stop being + // exercised. multiRunner := &multiCallRunner{ responses: []multiCallResponse{ + // database snapshot → success + {binary: "restic", argPrefix: "backup", err: nil}, // stack-a backup call → error {binary: "restic", argPrefix: "backup", err: errors.New("disk full")}, // stack-b backup call → success @@ -1864,3 +1873,167 @@ func TestRunRestore_HappyPath_EscapingSubdirIsRejected(t *testing.T) { } } } + +// ============================================================ +// Database snapshot in every backup run (agent-os-36o) +// ============================================================ + +// findCall returns the first recorded call whose args start with subcommand and +// contain every string in mustContain. +func findCall(calls []fakeCall, subcommand string, mustContain ...string) (fakeCall, bool) { + for _, c := range calls { + if len(c.Args) == 0 || c.Args[0] != subcommand { + continue + } + ok := true + for _, w := range mustContain { + found := false + for _, a := range c.Args { + if a == w { + found = true + break + } + } + if !found { + ok = false + break + } + } + if ok { + return c, true + } + } + return fakeCall{}, false +} + +// TestRunBackup_CapturesDatabaseUnderItsOwnTag pins the core of agent-os-36o: +// capstan.db is captured on every run, under a tag that is not a stack ID. +// +// Before this, the backup engine's scope was each stack's compose directory and +// the database — accounts, encrypted git tokens and restic password, settings, +// policies, history — was in no snapshot at all. +func TestRunBackup_CapturesDatabaseUnderItsOwnTag(t *testing.T) { + t.Parallel() + + db := newBackupTestDB(t) + runner := &fakeRunner{outputData: snapshotJSON("abc123", "abc", "myapp")} + svc := buildSvc(t, db, &fakeDocker{}, runner, runner) + seedStack(t, db, "myapp", "hot") + + out := make(chan StreamLine, 256) + run, err := svc.RunBackup(context.Background(), nil, false, TriggerManual, out) + require.NoError(t, err) + assert.Equal(t, "success", run.Status) + + call, ok := findCall(runner.calls, "backup", "--tag", DatabaseBackupTag) + require.True(t, ok, "every backup run must capture capstan.db under the %q tag", DatabaseBackupTag) + + assert.Contains(t, call.Args, svc.DatabaseSnapshotPath(), + "the database snapshot must be taken from the staged VACUUM INTO copy") + + // The tag must not collide with a stack ID, or the snapshot becomes + // selectable as if it were a stack. + assert.NotContains(t, call.Args, "myapp", + "the database snapshot must not carry a stack tag") +} + +// TestRunBackup_StagedDatabaseCopyIsRemoved verifies the staged artifact does +// not outlive the run. It is a full plaintext database sitting on disk. +func TestRunBackup_StagedDatabaseCopyIsRemoved(t *testing.T) { + t.Parallel() + + db := newBackupTestDB(t) + runner := &fakeRunner{outputData: snapshotJSON("abc123", "abc", "myapp")} + svc := buildSvc(t, db, &fakeDocker{}, runner, runner) + seedStack(t, db, "myapp", "hot") + + out := make(chan StreamLine, 256) + _, err := svc.RunBackup(context.Background(), nil, false, TriggerManual, out) + require.NoError(t, err) + + _, statErr := os.Stat(svc.DatabaseSnapshotPath()) + assert.True(t, os.IsNotExist(statErr), + "the staged plaintext database copy must be removed after the run, got err=%v", statErr) +} + +// TestRunBackup_DryRunDoesNotStageDatabase verifies a dry run neither writes the +// staged copy nor hands anything to restic. +func TestRunBackup_DryRunDoesNotStageDatabase(t *testing.T) { + t.Parallel() + + db := newBackupTestDB(t) + runner := &fakeRunner{outputData: snapshotJSON("abc123", "abc", "myapp")} + svc := buildSvc(t, db, &fakeDocker{}, runner, runner) + seedStack(t, db, "myapp", "hot") + + out := make(chan StreamLine, 256) + _, err := svc.RunBackup(context.Background(), nil, true, TriggerManual, out) + require.NoError(t, err) + + _, ok := findCall(runner.calls, "backup", "--tag", DatabaseBackupTag) + assert.False(t, ok, "a dry run must not hand the database to restic") + + _, statErr := os.Stat(svc.DatabaseSnapshotPath()) + assert.True(t, os.IsNotExist(statErr), "a dry run must not stage a database copy") +} + +// databaseFailingRunner fails only the restic invocation that carries the +// database tag, so a test can isolate a database-snapshot failure from a +// stack-backup failure. +type databaseFailingRunner struct { + fakeRunner +} + +func (f *databaseFailingRunner) Run( + ctx context.Context, + name string, + args []string, + env []string, + out chan<- StreamLine, +) error { + for _, a := range args { + if a == DatabaseBackupTag { + f.fakeRunner.calls = append(f.fakeRunner.calls, fakeCall{Binary: name, Args: args, Env: env}) + return errors.New("injected database snapshot failure") + } + } + return f.fakeRunner.Run(ctx, name, args, env, out) +} + +// TestRunBackup_DatabaseFailureDowngradesSuccessToPartial pins the rule that a +// run which saved every stack but lost the database is NOT reported as success. +// +// Without this, the most important artifact could go missing while the UI stayed +// green — the silent failure agent-os-36o exists to remove. +func TestRunBackup_DatabaseFailureDowngradesSuccessToPartial(t *testing.T) { + t.Parallel() + + db := newBackupTestDB(t) + runner := &databaseFailingRunner{fakeRunner: fakeRunner{outputData: snapshotJSON("abc123", "abc", "myapp")}} + svc := buildSvc(t, db, &fakeDocker{}, runner, runner) + seedStack(t, db, "myapp", "hot") + + out := make(chan StreamLine, 256) + run, err := svc.RunBackup(context.Background(), nil, false, TriggerManual, out) + require.NoError(t, err) + + assert.Equal(t, 1, run.StacksOK, "the stack backup itself should have succeeded") + assert.Equal(t, 0, run.StacksFailed) + assert.Equal(t, "partial", run.Status, + "a run that lost the database must not report success") + assert.NotEmpty(t, run.ErrorMessage, "the run must say why it was downgraded") +} + +// TestDatabaseSnapshotPath_IsUnderDataDir pins the path the runbook names. An +// operator restoring under pressure reads this path out of README; if it moves +// silently, the runbook is wrong at the worst possible moment. +func TestDatabaseSnapshotPath_IsUnderDataDir(t *testing.T) { + t.Parallel() + + db := newBackupTestDB(t) + svc := buildSvc(t, db, &fakeDocker{}, &fakeRunner{}, &fakeRunner{}) + + assert.Equal(t, + filepath.Join(svc.Config().DataDir, "backup-staging", "capstan.db"), + svc.DatabaseSnapshotPath()) +}