Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ and `sqlite` don't.

- **Read-only against your repository.** The agent shells out to *your*
restic and can only run `version`, `snapshots`, `ls`, `restore`, `check`,
`dump` — the whitelist is enforced in the type system; `forget`/`prune`
`dump`, `cat`. The whitelist is enforced in the type system; `forget`/`prune`
are a compile error, not a code-review promise.
- **Sandboxes are always destroyed** — on success, failure, and panic. Disk
space is checked *before* any restore begins.
Expand All @@ -115,20 +115,25 @@ and `sqlite` don't.

The hosted dashboard adds run history across repos, alerting on failure,
stale-backup detection (fires when your *testing* silently dies — the
meta-failure). Free tier: 1 repo, monthly tests, 1 alert channel. Pro
($8/mo or $80/yr): unlimited repos, any schedule, all channels, 12-month
history (monthly confidence reports coming soon). The agent works forever
without it.
meta-failure).

**It is all free during the open beta.** Every Pro feature, no card, no
trial clock. The plans below are what it will cost when the beta ends.
Free: 1 repo, monthly tests, 1 alert channel. Pro ($8/mo or $80/yr):
unlimited repos, any schedule, all channels, 12-month history. The agent
works forever without any of it.

## Docs

[Quickstart](docs/quickstart.md) ·
[Configuration](docs/agent-configuration.md) ·
[Commands](docs/commands.md) ·
[files](docs/recipes-files.md) ·
[databases](docs/recipes-databases.md) ·
[docker-app](docs/recipes-docker-app.md) ·
[Alerts](docs/alerts.md) ·
[Cloud](docs/cloud.md)
[Cloud](docs/cloud.md) ·
[Plans](docs/billing.md)

## Building from source

Expand Down
7 changes: 6 additions & 1 deletion agent/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cmd

import (
"errors"
"fmt"
"log"
"os"

Expand Down Expand Up @@ -52,7 +53,11 @@ func newRunCmd() *cobra.Command {
logger.Printf("write result: %v", err)
}
} else {
cmd.Println(res.Human())
// The result goes to stdout, progress to stderr, so `restorable test
// >> log` captures the verdict. cmd.Println writes to stderr.
if _, err := fmt.Fprintln(cmd.OutOrStdout(), res.Human()); err != nil {
logger.Printf("write result: %v", err)
}
}
logger.Printf("restore test finished: %s", res.Status)
}
Expand Down
8 changes: 7 additions & 1 deletion agent/cmd/test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package cmd

import (
"fmt"

"github.com/spf13/cobra"

"github.com/restorable-dev/restorable/agent/internal/config"
Expand Down Expand Up @@ -49,7 +51,11 @@ func newTestCmd() *cobra.Command {
return err
}
} else {
cmd.Println(res.Human())
// The result goes to stdout, progress to stderr, so `restorable test
// >> log` captures the verdict. cmd.Println writes to stderr.
if _, err := fmt.Fprintln(cmd.OutOrStdout(), res.Human()); err != nil {
return err
}
}
if code := res.ExitCode(); code != 0 {
return &exitError{code: code}
Expand Down
88 changes: 88 additions & 0 deletions agent/e2e/run-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -137,4 +137,92 @@ PY
assert_docker_clean "failing run"
assert_sandbox_empty


# ── docker-app ───────────────────────────────────────────────────────────────
# docker-app shipped broken in v0.1.0 and v0.1.1 and nothing caught it: its
# unit test uses a fake ContainerRunner, so the real StartContainer and
# MappedPort path had never executed in CI. These cases run it for real.

log "case 3: docker-app boots the real image against restored data (exit 0)"
APP_SRC="$WORK/src-app"
mkdir -p "$APP_SRC/site"
echo "<html><body>restored-ok</body></html>" > "$APP_SRC/site/index.html"
APP_REPO="$WORK/app-repo"
restic -r "$APP_REPO" init -q
restic -r "$APP_REPO" backup -q "$APP_SRC"

cat > "$WORK/recipe-app.yaml" <<EOF
name: e2e-docker-app
checks:
- type: docker-app
image: nginx:alpine
mount: { restored: "site", at: "/usr/share/nginx/html" }
ready:
http: "http://localhost:80/index.html"
contains: "restored-ok"
timeout: 90s
EOF
cat > "$WORK/agent-app.yaml" <<EOF
repo: $APP_REPO
recipes: [recipe-app.yaml]
sandbox:
dir: $SANDBOX
EOF

"$BIN" test --config "$WORK/agent-app.yaml" --json > "$WORK/app.json"
python3 - "$WORK/app.json" <<'PYEOF'
import json, sys
doc = json.load(open(sys.argv[1]))
assert doc["status"] == "pass", doc
[check] = doc["checks"]
assert check["type"] == "docker-app", check
assert check["status"] == "pass", check
PYEOF
assert_docker_clean "docker-app passing run"
assert_sandbox_empty

log "case 4: docker-app fails when restored data lacks the asserted page (exit 1)"
APP_BAD_SRC="$WORK/src-app-bad"
mkdir -p "$APP_BAD_SRC/site"
echo "placeholder" > "$APP_BAD_SRC/site/other.html" # index.html absent
APP_BAD_REPO="$WORK/app-bad-repo"
restic -r "$APP_BAD_REPO" init -q
restic -r "$APP_BAD_REPO" backup -q "$APP_BAD_SRC"

cat > "$WORK/recipe-app-bad.yaml" <<EOF
name: e2e-docker-app-bad
checks:
- type: docker-app
image: nginx:alpine
mount: { restored: "site", at: "/usr/share/nginx/html" }
ready:
http: "http://localhost:80/index.html"
contains: "restored-ok"
timeout: 30s
EOF
cat > "$WORK/agent-app-bad.yaml" <<EOF
repo: $APP_BAD_REPO
recipes: [recipe-app-bad.yaml]
sandbox:
dir: $SANDBOX
EOF

got=0
"$BIN" test --config "$WORK/agent-app-bad.yaml" --json > "$WORK/app-bad.json" || got=$?
[ "$got" -eq 1 ] || fail "expected exit 1 for docker-app with missing page, got $got"
python3 - "$WORK/app-bad.json" <<'PYEOF'
import json, sys
doc = json.load(open(sys.argv[1]))
assert doc["status"] == "fail", doc
[check] = doc["checks"]
assert check["status"] == "fail", check
msg = check["message"]
# Must fail on readiness, never on "not published". That message meant the
# port lookup lost its race with the daemon and blamed the wrong thing.
assert "not published" not in msg, f"port race regressed: {msg!r}"
assert "not ready" in msg or "exited early" in msg, f"unclear message: {msg!r}"
PYEOF
assert_docker_clean "docker-app failing run"
assert_sandbox_empty

log "all docker e2e cases passed"
6 changes: 6 additions & 0 deletions agent/internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ var envNameRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
// resolved relative to the config file's directory.
func Load(path string) (*Config, error) {
f, err := os.Open(path)
if errors.Is(err, os.ErrNotExist) {
// The first thing a new user hits if they skip straight to `test`.
// Name the command that creates the file rather than leaving them to
// find it in the docs.
return nil, fmt.Errorf("no config at %s — run `restorable init --repo <your-repo>` to create one", path)
}
if err != nil {
return nil, fmt.Errorf("open config: %w", err)
}
Expand Down
25 changes: 25 additions & 0 deletions agent/internal/recipes/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"io"
"os"
"regexp"
"strings"
"time"

Expand Down Expand Up @@ -160,6 +161,30 @@ func openDump(path, tmpDir string) (r io.ReadCloser, size int64, custom bool, er
return f, info.Size(), string(head) == "PGDMP", nil
}

// errLineRe finds the line carrying the actual reason a database command
// failed.
var errLineRe = regexp.MustCompile(`(?i)\b(ERROR|FATAL)\b`)

// diagnostic summarises command output for a user-facing message, starting
// from the line that says what went wrong.
//
// tail() alone was wrong here: psql reports the reason first and then echoes
// the offending statement with a caret under it, so keeping the last two lines
// kept the caret and discarded "relation \"assets\" does not exist". Users got
// an arrow pointing at nothing, and the quickstart promises the message names
// exactly what was missing.
func diagnostic(s string, n int) string {
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
for i, line := range lines {
if !errLineRe.MatchString(line) {
continue
}
end := min(i+n, len(lines))
return strings.Join(lines[i:end], " / ")
}
return tail(s, n)
}

// tail returns the last n lines of command output for error messages.
func tail(s string, n int) string {
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
Expand Down
62 changes: 62 additions & 0 deletions agent/internal/recipes/diagnostic_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package recipes

import (
"strings"
"testing"
)

// psql and mysql print the reason first and then echo the offending statement
// with a caret under it. tail() kept the caret and dropped the reason, so
// users saw an arrow pointing at nothing while the sentence explaining the
// failure was discarded.
func TestDiagnosticKeepsTheReason(t *testing.T) {
tests := []struct {
name string
out string
n int
mustHave string
}{
{
name: "postgres missing relation",
out: "ERROR: relation \"assets\" does not exist\n" +
"LINE 1: SELECT count(*) FROM \"assets\"\n" +
" ^",
n: 2,
mustHave: `relation "assets" does not exist`,
},
{
name: "reason preceded by noise",
out: "Pager usage is off.\n" +
"psql:/tmp/dump:3: ERROR: syntax error at or near \")\"\n" +
"LINE 1: ...\n" +
" ^",
n: 2,
mustHave: "syntax error at or near",
},
{
name: "mysql fatal",
out: "mysql: [Warning] Using a password on the command line\nERROR 1146 (42S02) at line 3: Table 'restorable.users' doesn't exist",
n: 2,
mustHave: "doesn't exist",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := diagnostic(tt.out, tt.n)
if !strings.Contains(got, tt.mustHave) {
t.Errorf("dropped the reason\n want substring: %s\n got: %s", tt.mustHave, got)
}
})
}
}

// With nothing error-shaped to anchor on, fall back to the old behaviour
// rather than returning nothing.
func TestDiagnosticFallsBackToTail(t *testing.T) {
out := "line one\nline two\nline three"
got := diagnostic(out, 2)
if !strings.Contains(got, "line three") {
t.Errorf("expected tail fallback, got %q", got)
}
}
4 changes: 2 additions & 2 deletions agent/internal/recipes/mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func (c *MySQLCheck) Run(ctx context.Context, t *Target) (report.Status, string)
return report.StatusError, err.Error()
}
if code != 0 {
return report.StatusFail, fmt.Sprintf("dump %q failed to load into mysql: %s", c.Dump, tail(out, 3))
return report.StatusFail, fmt.Sprintf("dump %q failed to load into mysql: %s", c.Dump, diagnostic(out, 3))
}

for _, tbl := range c.Tables {
Expand All @@ -112,7 +112,7 @@ func (c *MySQLCheck) Run(ctx context.Context, t *Target) (report.Status, string)
return report.StatusError, err.Error()
}
if code != 0 {
return report.StatusFail, fmt.Sprintf("count rows in %q: %s", tbl.Name, tail(out, 2))
return report.StatusFail, fmt.Sprintf("count rows in %q: %s", tbl.Name, diagnostic(out, 2))
}
n, err := strconv.ParseInt(strings.TrimSpace(stripMySQLWarning(out)), 10, 64)
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions agent/internal/recipes/postgres.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ func (c *PostgresCheck) Run(ctx context.Context, t *Target) (report.Status, stri
return report.StatusError, err.Error()
}
if code != 0 {
return report.StatusFail, fmt.Sprintf("dump %q failed to load into postgres: %s", c.Dump, tail(out, 3))
return report.StatusFail, fmt.Sprintf("dump %q failed to load into postgres: %s", c.Dump, diagnostic(out, 3))
}

for _, tbl := range c.Tables {
Expand All @@ -152,7 +152,7 @@ func (c *PostgresCheck) Run(ctx context.Context, t *Target) (report.Status, stri
return report.StatusError, err.Error()
}
if code != 0 {
return report.StatusFail, fmt.Sprintf("count rows in %q: %s", tbl.Name, tail(out, 2))
return report.StatusFail, fmt.Sprintf("count rows in %q: %s", tbl.Name, diagnostic(out, 2))
}
n, err := strconv.ParseInt(strings.TrimSpace(out), 10, 64)
if err != nil {
Expand Down
26 changes: 25 additions & 1 deletion agent/internal/restic/restic.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func (r *Runner) run(ctx context.Context, sub subcommand, out io.Writer, args ..
cmd.Stderr = &stderr
cmd.Env = r.env()
if err := cmd.Run(); err != nil {
msg := strings.TrimSpace(stderr.String())
msg := resticMessage(strings.TrimSpace(stderr.String()))
if msg == "" {
return fmt.Errorf("restic %s: %w", sub, err)
}
Expand All @@ -95,6 +95,30 @@ func (r *Runner) run(ctx context.Context, sub subcommand, out io.Writer, args ..
return nil
}

// resticMessage pulls the readable sentence out of restic's error output.
//
// Several subcommands are invoked with --json, and restic then reports
// failures as {"message_type":"exit_error","code":12,"message":"Fatal: wrong
// password or no key found"}. A wrong password or a mistyped repo path is the
// most common first-run mistake there is, so showing someone the raw object
// and burying the one sentence that tells them what to fix is the worst place
// to do it.
func resticMessage(stderr string) string {
for _, line := range strings.Split(stderr, "\n") {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "{") {
continue
}
var e struct {
Message string `json:"message"`
}
if json.Unmarshal([]byte(line), &e) == nil && e.Message != "" {
return e.Message
}
}
return stderr
}

// Version returns the restic version string.
func (r *Runner) Version(ctx context.Context) (string, error) {
var out bytes.Buffer
Expand Down
Loading
Loading