diff --git a/README.md b/README.md index f2ac9f9..d4f14f9 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ and `sqlite` don't. - **Read-only against your repository.** The agent shells out to *your* restic and can only run `version`, `snapshots`, `ls`, `restore`, `check`, - `dump` — the whitelist is enforced in the type system; `forget`/`prune` + `dump`, `cat`. The whitelist is enforced in the type system; `forget`/`prune` are a compile error, not a code-review promise. - **Sandboxes are always destroyed** — on success, failure, and panic. Disk space is checked *before* any restore begins. @@ -115,20 +115,25 @@ and `sqlite` don't. The hosted dashboard adds run history across repos, alerting on failure, stale-backup detection (fires when your *testing* silently dies — the -meta-failure). Free tier: 1 repo, monthly tests, 1 alert channel. Pro -($8/mo or $80/yr): unlimited repos, any schedule, all channels, 12-month -history (monthly confidence reports coming soon). The agent works forever -without it. +meta-failure). + +**It is all free during the open beta.** Every Pro feature, no card, no +trial clock. The plans below are what it will cost when the beta ends. +Free: 1 repo, monthly tests, 1 alert channel. Pro ($8/mo or $80/yr): +unlimited repos, any schedule, all channels, 12-month history. The agent +works forever without any of it. ## Docs +[Quickstart](docs/quickstart.md) · [Configuration](docs/agent-configuration.md) · [Commands](docs/commands.md) · [files](docs/recipes-files.md) · [databases](docs/recipes-databases.md) · [docker-app](docs/recipes-docker-app.md) · [Alerts](docs/alerts.md) · -[Cloud](docs/cloud.md) +[Cloud](docs/cloud.md) · +[Plans](docs/billing.md) ## Building from source diff --git a/agent/cmd/run.go b/agent/cmd/run.go index 308403c..4099465 100644 --- a/agent/cmd/run.go +++ b/agent/cmd/run.go @@ -2,6 +2,7 @@ package cmd import ( "errors" + "fmt" "log" "os" @@ -52,7 +53,11 @@ func newRunCmd() *cobra.Command { logger.Printf("write result: %v", err) } } else { - cmd.Println(res.Human()) + // The result goes to stdout, progress to stderr, so `restorable test + // >> log` captures the verdict. cmd.Println writes to stderr. + if _, err := fmt.Fprintln(cmd.OutOrStdout(), res.Human()); err != nil { + logger.Printf("write result: %v", err) + } } logger.Printf("restore test finished: %s", res.Status) } diff --git a/agent/cmd/test.go b/agent/cmd/test.go index b71aca6..0585c03 100644 --- a/agent/cmd/test.go +++ b/agent/cmd/test.go @@ -1,6 +1,8 @@ package cmd import ( + "fmt" + "github.com/spf13/cobra" "github.com/restorable-dev/restorable/agent/internal/config" @@ -49,7 +51,11 @@ func newTestCmd() *cobra.Command { return err } } else { - cmd.Println(res.Human()) + // The result goes to stdout, progress to stderr, so `restorable test + // >> log` captures the verdict. cmd.Println writes to stderr. + if _, err := fmt.Fprintln(cmd.OutOrStdout(), res.Human()); err != nil { + return err + } } if code := res.ExitCode(); code != 0 { return &exitError{code: code} diff --git a/agent/e2e/run-docker.sh b/agent/e2e/run-docker.sh index cb0d36d..7dc68e0 100755 --- a/agent/e2e/run-docker.sh +++ b/agent/e2e/run-docker.sh @@ -137,4 +137,92 @@ PY assert_docker_clean "failing run" assert_sandbox_empty + +# ── docker-app ─────────────────────────────────────────────────────────────── +# docker-app shipped broken in v0.1.0 and v0.1.1 and nothing caught it: its +# unit test uses a fake ContainerRunner, so the real StartContainer and +# MappedPort path had never executed in CI. These cases run it for real. + +log "case 3: docker-app boots the real image against restored data (exit 0)" +APP_SRC="$WORK/src-app" +mkdir -p "$APP_SRC/site" +echo "restored-ok" > "$APP_SRC/site/index.html" +APP_REPO="$WORK/app-repo" +restic -r "$APP_REPO" init -q +restic -r "$APP_REPO" backup -q "$APP_SRC" + +cat > "$WORK/recipe-app.yaml" < "$WORK/agent-app.yaml" < "$WORK/app.json" +python3 - "$WORK/app.json" <<'PYEOF' +import json, sys +doc = json.load(open(sys.argv[1])) +assert doc["status"] == "pass", doc +[check] = doc["checks"] +assert check["type"] == "docker-app", check +assert check["status"] == "pass", check +PYEOF +assert_docker_clean "docker-app passing run" +assert_sandbox_empty + +log "case 4: docker-app fails when restored data lacks the asserted page (exit 1)" +APP_BAD_SRC="$WORK/src-app-bad" +mkdir -p "$APP_BAD_SRC/site" +echo "placeholder" > "$APP_BAD_SRC/site/other.html" # index.html absent +APP_BAD_REPO="$WORK/app-bad-repo" +restic -r "$APP_BAD_REPO" init -q +restic -r "$APP_BAD_REPO" backup -q "$APP_BAD_SRC" + +cat > "$WORK/recipe-app-bad.yaml" < "$WORK/agent-app-bad.yaml" < "$WORK/app-bad.json" || got=$? +[ "$got" -eq 1 ] || fail "expected exit 1 for docker-app with missing page, got $got" +python3 - "$WORK/app-bad.json" <<'PYEOF' +import json, sys +doc = json.load(open(sys.argv[1])) +assert doc["status"] == "fail", doc +[check] = doc["checks"] +assert check["status"] == "fail", check +msg = check["message"] +# Must fail on readiness, never on "not published". That message meant the +# port lookup lost its race with the daemon and blamed the wrong thing. +assert "not published" not in msg, f"port race regressed: {msg!r}" +assert "not ready" in msg or "exited early" in msg, f"unclear message: {msg!r}" +PYEOF +assert_docker_clean "docker-app failing run" +assert_sandbox_empty + log "all docker e2e cases passed" diff --git a/agent/internal/config/config.go b/agent/internal/config/config.go index c4e8f46..1dcc68d 100644 --- a/agent/internal/config/config.go +++ b/agent/internal/config/config.go @@ -52,6 +52,12 @@ var envNameRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) // resolved relative to the config file's directory. func Load(path string) (*Config, error) { f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + // The first thing a new user hits if they skip straight to `test`. + // Name the command that creates the file rather than leaving them to + // find it in the docs. + return nil, fmt.Errorf("no config at %s — run `restorable init --repo ` to create one", path) + } if err != nil { return nil, fmt.Errorf("open config: %w", err) } diff --git a/agent/internal/recipes/container.go b/agent/internal/recipes/container.go index cb59eef..a6cce6c 100644 --- a/agent/internal/recipes/container.go +++ b/agent/internal/recipes/container.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "regexp" "strings" "time" @@ -160,6 +161,30 @@ func openDump(path, tmpDir string) (r io.ReadCloser, size int64, custom bool, er return f, info.Size(), string(head) == "PGDMP", nil } +// errLineRe finds the line carrying the actual reason a database command +// failed. +var errLineRe = regexp.MustCompile(`(?i)\b(ERROR|FATAL)\b`) + +// diagnostic summarises command output for a user-facing message, starting +// from the line that says what went wrong. +// +// tail() alone was wrong here: psql reports the reason first and then echoes +// the offending statement with a caret under it, so keeping the last two lines +// kept the caret and discarded "relation \"assets\" does not exist". Users got +// an arrow pointing at nothing, and the quickstart promises the message names +// exactly what was missing. +func diagnostic(s string, n int) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + for i, line := range lines { + if !errLineRe.MatchString(line) { + continue + } + end := min(i+n, len(lines)) + return strings.Join(lines[i:end], " / ") + } + return tail(s, n) +} + // tail returns the last n lines of command output for error messages. func tail(s string, n int) string { lines := strings.Split(strings.TrimRight(s, "\n"), "\n") diff --git a/agent/internal/recipes/diagnostic_test.go b/agent/internal/recipes/diagnostic_test.go new file mode 100644 index 0000000..3516119 --- /dev/null +++ b/agent/internal/recipes/diagnostic_test.go @@ -0,0 +1,62 @@ +package recipes + +import ( + "strings" + "testing" +) + +// psql and mysql print the reason first and then echo the offending statement +// with a caret under it. tail() kept the caret and dropped the reason, so +// users saw an arrow pointing at nothing while the sentence explaining the +// failure was discarded. +func TestDiagnosticKeepsTheReason(t *testing.T) { + tests := []struct { + name string + out string + n int + mustHave string + }{ + { + name: "postgres missing relation", + out: "ERROR: relation \"assets\" does not exist\n" + + "LINE 1: SELECT count(*) FROM \"assets\"\n" + + " ^", + n: 2, + mustHave: `relation "assets" does not exist`, + }, + { + name: "reason preceded by noise", + out: "Pager usage is off.\n" + + "psql:/tmp/dump:3: ERROR: syntax error at or near \")\"\n" + + "LINE 1: ...\n" + + " ^", + n: 2, + mustHave: "syntax error at or near", + }, + { + name: "mysql fatal", + out: "mysql: [Warning] Using a password on the command line\nERROR 1146 (42S02) at line 3: Table 'restorable.users' doesn't exist", + n: 2, + mustHave: "doesn't exist", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := diagnostic(tt.out, tt.n) + if !strings.Contains(got, tt.mustHave) { + t.Errorf("dropped the reason\n want substring: %s\n got: %s", tt.mustHave, got) + } + }) + } +} + +// With nothing error-shaped to anchor on, fall back to the old behaviour +// rather than returning nothing. +func TestDiagnosticFallsBackToTail(t *testing.T) { + out := "line one\nline two\nline three" + got := diagnostic(out, 2) + if !strings.Contains(got, "line three") { + t.Errorf("expected tail fallback, got %q", got) + } +} diff --git a/agent/internal/recipes/mysql.go b/agent/internal/recipes/mysql.go index 1d7fd85..bea612c 100644 --- a/agent/internal/recipes/mysql.go +++ b/agent/internal/recipes/mysql.go @@ -101,7 +101,7 @@ func (c *MySQLCheck) Run(ctx context.Context, t *Target) (report.Status, string) return report.StatusError, err.Error() } if code != 0 { - return report.StatusFail, fmt.Sprintf("dump %q failed to load into mysql: %s", c.Dump, tail(out, 3)) + return report.StatusFail, fmt.Sprintf("dump %q failed to load into mysql: %s", c.Dump, diagnostic(out, 3)) } for _, tbl := range c.Tables { @@ -112,7 +112,7 @@ func (c *MySQLCheck) Run(ctx context.Context, t *Target) (report.Status, string) return report.StatusError, err.Error() } if code != 0 { - return report.StatusFail, fmt.Sprintf("count rows in %q: %s", tbl.Name, tail(out, 2)) + return report.StatusFail, fmt.Sprintf("count rows in %q: %s", tbl.Name, diagnostic(out, 2)) } n, err := strconv.ParseInt(strings.TrimSpace(stripMySQLWarning(out)), 10, 64) if err != nil { diff --git a/agent/internal/recipes/postgres.go b/agent/internal/recipes/postgres.go index ba97eb8..2c44861 100644 --- a/agent/internal/recipes/postgres.go +++ b/agent/internal/recipes/postgres.go @@ -142,7 +142,7 @@ func (c *PostgresCheck) Run(ctx context.Context, t *Target) (report.Status, stri return report.StatusError, err.Error() } if code != 0 { - return report.StatusFail, fmt.Sprintf("dump %q failed to load into postgres: %s", c.Dump, tail(out, 3)) + return report.StatusFail, fmt.Sprintf("dump %q failed to load into postgres: %s", c.Dump, diagnostic(out, 3)) } for _, tbl := range c.Tables { @@ -152,7 +152,7 @@ func (c *PostgresCheck) Run(ctx context.Context, t *Target) (report.Status, stri return report.StatusError, err.Error() } if code != 0 { - return report.StatusFail, fmt.Sprintf("count rows in %q: %s", tbl.Name, tail(out, 2)) + return report.StatusFail, fmt.Sprintf("count rows in %q: %s", tbl.Name, diagnostic(out, 2)) } n, err := strconv.ParseInt(strings.TrimSpace(out), 10, 64) if err != nil { diff --git a/agent/internal/restic/restic.go b/agent/internal/restic/restic.go index 71cc095..43ebaef 100644 --- a/agent/internal/restic/restic.go +++ b/agent/internal/restic/restic.go @@ -86,7 +86,7 @@ func (r *Runner) run(ctx context.Context, sub subcommand, out io.Writer, args .. cmd.Stderr = &stderr cmd.Env = r.env() if err := cmd.Run(); err != nil { - msg := strings.TrimSpace(stderr.String()) + msg := resticMessage(strings.TrimSpace(stderr.String())) if msg == "" { return fmt.Errorf("restic %s: %w", sub, err) } @@ -95,6 +95,30 @@ func (r *Runner) run(ctx context.Context, sub subcommand, out io.Writer, args .. return nil } +// resticMessage pulls the readable sentence out of restic's error output. +// +// Several subcommands are invoked with --json, and restic then reports +// failures as {"message_type":"exit_error","code":12,"message":"Fatal: wrong +// password or no key found"}. A wrong password or a mistyped repo path is the +// most common first-run mistake there is, so showing someone the raw object +// and burying the one sentence that tells them what to fix is the worst place +// to do it. +func resticMessage(stderr string) string { + for _, line := range strings.Split(stderr, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "{") { + continue + } + var e struct { + Message string `json:"message"` + } + if json.Unmarshal([]byte(line), &e) == nil && e.Message != "" { + return e.Message + } + } + return stderr +} + // Version returns the restic version string. func (r *Runner) Version(ctx context.Context) (string, error) { var out bytes.Buffer diff --git a/agent/internal/sandbox/docker.go b/agent/internal/sandbox/docker.go index 7f34689..6c0179b 100644 --- a/agent/internal/sandbox/docker.go +++ b/agent/internal/sandbox/docker.go @@ -24,6 +24,14 @@ import ( // still runs when the surrounding run is cancelled (SIGTERM, timeout). const cleanupTimeout = 60 * time.Second +// How long to wait for the daemon to publish a container's port, and how often +// to re-check. Publication is asynchronous, so the binding is routinely absent +// for a moment after start; a single inspect loses that race every time. +const ( + portPublishTimeout = 15 * time.Second + portPollInterval = 100 * time.Millisecond +) + // ContainerSpec describes a throwaway container for a verification check. type ContainerSpec struct { Image string @@ -93,7 +101,9 @@ func (d *Docker) StartContainer(ctx context.Context, spec ContainerSpec) (string } host := &container.HostConfig{Binds: spec.Binds} if spec.PublishPort != "" { - port, err := nat.NewPort(strings.TrimSuffix(spec.PublishPort, "/tcp"), "tcp") + // NewPort takes the protocol first, then the port. Passing them the + // other way round parses "tcp" as a port number and fails every time. + port, err := nat.NewPort("tcp", strings.TrimSuffix(spec.PublishPort, "/tcp")) if err != nil { return "", fmt.Errorf("invalid port %q: %w", spec.PublishPort, err) } @@ -184,16 +194,36 @@ func (d *Docker) CopyTo(ctx context.Context, id, dstDir, name string, content io // MappedPort returns the ephemeral host port bound to the given container // port ("8080/tcp"). func (d *Docker) MappedPort(ctx context.Context, id, containerPort string) (string, error) { - inspect, err := d.cli.ContainerInspect(ctx, id) - if err != nil { - return "", fmt.Errorf("inspect container: %w", err) - } port := nat.Port(containerPort) - bindings := inspect.NetworkSettings.Ports[port] - if len(bindings) == 0 { - return "", fmt.Errorf("container port %s is not published", containerPort) + deadline := time.Now().Add(portPublishTimeout) + for { + inspect, err := d.cli.ContainerInspect(ctx, id) + if err != nil { + return "", fmt.Errorf("inspect container: %w", err) + } + // The daemon fills NetworkSettings.Ports asynchronously, so an inspect + // issued straight after start reliably returns an empty binding even + // though HostConfig already carries it. Poll rather than read once. + if b := inspect.NetworkSettings.Ports[port]; len(b) > 0 && b[0].HostPort != "" { + return b[0].HostPort, nil + } + // An app that dies on the restored data never publishes anything. That + // is the common real failure here, so name it instead of blaming the + // port and sending the user looking in the wrong place. + if !inspect.State.Running { + return "", fmt.Errorf("container exited early with code %d before publishing port %s", + inspect.State.ExitCode, containerPort) + } + if time.Now().After(deadline) { + return "", fmt.Errorf("container port %s was not published within %s", + containerPort, portPublishTimeout) + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(portPollInterval): + } } - return bindings[0].HostPort, nil } // State reports whether the container is still running, and its exit code diff --git a/docs/agent-configuration.md b/docs/agent-configuration.md index 113d2c4..f64cbdf 100644 --- a/docs/agent-configuration.md +++ b/docs/agent-configuration.md @@ -47,6 +47,7 @@ the test aborts with a clear error and nothing is restored. ## Read-only guarantee The agent only ever runs read operations against your repository: `version`, -`snapshots`, `ls`, `restore`, `check`, `dump`. This whitelist is enforced in +`snapshots`, `ls`, `restore`, `check`, `dump`, `cat`. This whitelist is +enforced in the code's type system — the exec wrapper physically cannot run `forget`, `prune`, or anything else. diff --git a/docs/alerts.md b/docs/alerts.md index 3eb1167..8f91ba0 100644 --- a/docs/alerts.md +++ b/docs/alerts.md @@ -43,5 +43,8 @@ before it receives real alerts. ## Cron -Stale and silent detection run from an hourly scheduled invocation of -`/api/cron/alerts` (`vercel.json`), authenticated with `CRON_SECRET`. +Stale and silent detection run from `/api/cron/alerts`, authenticated with +`CRON_SECRET`. Two schedulers call it: a GitHub Actions workflow +(`.github/workflows/cron-alerts.yml`) hourly, and a Vercel cron +(`vercel.json`) daily as a backstop. If alerts seem late, check the Actions +workflow first: scheduled workflows get disabled after repository inactivity. diff --git a/docs/billing.md b/docs/billing.md index 536c62f..d19e17e 100644 --- a/docs/billing.md +++ b/docs/billing.md @@ -1,5 +1,10 @@ # Plans & billing +> **Open beta: everything here is free right now.** Every Pro feature is +> unlocked for every account, no card and no trial clock. The limits and +> prices below describe what happens when the beta ends, not what is +> enforced today. + | | Free | Pro ($8/mo or $80/yr) | |---|---|---| | Repositories | 1 | Unlimited | @@ -13,7 +18,7 @@ alert channel. Pro removes the limits and adds monthly confidence reports ## How limits behave -Limits are enforced server-side, at the moment of growth: +Once the beta ends, limits are enforced server-side, at the moment of growth: - A run for a **new** repository beyond your limit is rejected with a clear `plan limit` error. Repositories that already report are never blocked. diff --git a/docs/quickstart.md b/docs/quickstart.md index a7848e5..b303312 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -53,8 +53,12 @@ checks: checksum_sample: 5 # verify 5 random restored files against the repository ``` -Not sure what paths to assert? `restic ls latest | head -50` shows what's in -your latest snapshot. Backing up Nextcloud, Vaultwarden, or Immich? Copy a +Not sure what paths to assert? This shows what's in your latest snapshot: + +```sh +restic -r /srv/backups/restic ls latest | head -50 +``` + Backing up Nextcloud, Vaultwarden, or Immich? Copy a [ready-made recipe](../recipes/) instead. ## 3. Run your first test (1 min + restore time) @@ -85,10 +89,12 @@ the number that matters on the day you actually need it. ## 4. Make it automatic (2 min) -Cron (uses exit codes: 0 pass, 1 fail, 2 error): +Cron (uses exit codes: 0 pass, 1 fail, 2 error). Results go to stdout and +progress to stderr, so keep them apart and the results file stays one JSON +object per run: ``` -0 3 * * 0 cd /etc/restorable && RESTIC_PASSWORD_FILE=/etc/restorable/pw restorable test --json >> /var/log/restorable.log 2>&1 +0 3 * * 0 RESTIC_PASSWORD="$(cat /etc/restorable/pw)" restorable test --config /etc/restorable/agent.yaml --json >> /var/log/restorable.jsonl 2>> /var/log/restorable.log ``` Or run the built-in daemon — add `schedule: "0 3 * * 0"` to `agent.yaml` and: @@ -99,11 +105,12 @@ restorable run ## 5. Optional: alerts + dashboard (3 min) -Create an account on the dashboard, mint a registration token under +Create an account at [restorable.dev](https://restorable.dev), mint a +registration token under **Agents**, and connect: ```sh -restorable register --url https:// --token rrt_… +restorable register --url https://restorable.dev --token rrt_… ``` Every test now reports pass/fail metadata (never your data) to your diff --git a/recipes/immich.yaml b/recipes/immich.yaml index 88d4e31..08f987c 100644 --- a/recipes/immich.yaml +++ b/recipes/immich.yaml @@ -1,9 +1,7 @@ # Immich restore verification. # # Assumes your backup covers Immich's UPLOAD_LOCATION, which also holds the -# automatic database dumps Immich writes to backups/ (gzipped pg_dump files — -# supported directly). Adjust the dump path to your newest dump file, or -# point it at wherever your own pg_dump job writes. +# automatic database dumps Immich writes to backups/. name: immich checks: - type: files @@ -13,15 +11,25 @@ checks: - path: upload/ checksum_sample: 5 - # Immich's own DB dumps land in backups/ under the upload location. - # The dump filename contains a timestamp; point this at the one your - # backup captures (a stable symlink or copy step in your backup job - # makes this deterministic). - - type: postgres - dump: backups/immich-db-backup-latest.sql.gz - image: postgres:16-alpine - tables: - - name: users - min_rows: 1 - - name: assets - min_rows: 1 + # Immich names its dumps with a timestamp and two version numbers, e.g. + # backups/immich-db-backup-20250729T114018-v1.136.0-pg14.17.sql.gz + # so there is no fixed path to assert: the filename changes every run and + # again on every upgrade. Recipes are literal paths, with no globbing. + # + # To verify the database too, give your backup job a stable name to point + # at, then uncomment the check below and use that path: + # + # cp "$(ls -t /path/to/upload/backups/immich-db-backup-*.sql.gz | head -1)" \ + # /path/to/upload/backups/latest.sql.gz + # + # - type: postgres + # dump: backups/latest.sql.gz + # image: postgres:16-alpine + # tables: + # # Singular: Immich declares @Table('user') and @Table('asset'). + # # They were renamed from the plural forms in the 1.13x line, so check + # # your own dump if you are on something older. + # - name: user + # min_rows: 1 + # - name: asset + # min_rows: 1